diff --git a/FacebookAI/xlm-roberta-large/onnx/model.onnx b/FacebookAI/xlm-roberta-large/onnx/model.onnx new file mode 100644 index 0000000000000000000000000000000000000000..b4439b368fe07efacf7530d2a9e0244839a56c69 --- /dev/null +++ b/FacebookAI/xlm-roberta-large/onnx/model.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb5a52503a3ef35247f5b5ae6c473aaae60505dd3ffaef56d7b69e2f84683c05 +size 545850 diff --git a/mosesdecoder/contrib/mada/qsub-madamira.perl b/mosesdecoder/contrib/mada/qsub-madamira.perl new file mode 100644 index 0000000000000000000000000000000000000000..bb7ecd06b5c7642c40f379eabd3cf082a12ba12e --- /dev/null +++ b/mosesdecoder/contrib/mada/qsub-madamira.perl @@ -0,0 +1,46 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use File::Slurp; +use File::Basename; +use Cwd 'abs_path'; + +my $splitDir = $ARGV[0]; +$splitDir = abs_path($splitDir); + +my @files = read_dir $splitDir; + +my $qsubDir=dirname($splitDir) ."/qsub"; +print STDERR "qsubDir=$qsubDir\n"; +`mkdir -p $qsubDir`; + +my $out2Dir=dirname($splitDir) ."/out2"; +print STDERR "out2Dir=$out2Dir\n"; +`mkdir -p $out2Dir`; + +for my $file ( @files ) { + print STDERR "$file "; + + my $qsubFile = "$qsubDir/$file.sh"; + open(RUN_FILE, ">$qsubFile"); + + print RUN_FILE "#!/usr/bin/env bash\n" + ."#PBS -d/scratch/hh65/workspace/experiment/ar-en \n" + ."#PBS -l mem=5gb \n\n" + ."export PATH=\"/scratch/statmt/bin:/share/apps/NYUAD/perl/gcc_4.9.1/5.20.1/bin:/share/apps/NYUAD/jdk/1.8.0_31/bin:/share/apps/NYUAD/zlib/gcc_4.9.1/1.2.8/bin:/share/apps/NYUAD/cmake/gcc_4.9.1/3.1.0-rc3/bin:/share/apps/NYUAD/boost/gcc_4.9.1/openmpi_1.8.3/1.57.0/bin:/share/apps/NYUAD/openmpi/gcc_4.9.1/1.8.3/bin:/share/apps/NYUAD/python/gcc_4.9.1/2.7.9/bin:/share/apps/NYUAD/gcc/binutils/2.21/el6/bin:/share/apps/NYUAD/gcc/gcc/4.9.1/el6/bin:/usr/lib64/qt-3.3/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/bio/ncbi/bin:/opt/bio/mpiblast/bin:/opt/bio/EMBOSS/bin:/opt/bio/clustalw/bin:/opt/bio/tcoffee/bin:/opt/bio/hmmer/bin:/opt/bio/phylip/exe:/opt/bio/mrbayes:/opt/bio/fasta:/opt/bio/glimmer/bin:/opt/bio/glimmer/scripts:/opt/bio/gromacs/bin:/opt/bio/gmap/bin:/opt/bio/tigr/bin:/opt/bio/autodocksuite/bin:/opt/bio/wgs/bin:/opt/ganglia/bin:/opt/ganglia/sbin:/opt/bin:/usr/java/latest/bin:/opt/pdsh/bin:/opt/rocks/bin:/opt/rocks/sbin:/opt/torque/bin:/opt/torque/sbin:/home/hh65/bin:/home/hh65/bin\" \n" + + ."module load NYUAD/2.0 \n" + ."module load gcc python/2.7.9 openmpi/1.8.3 boost cmake zlib jdk perl expat \n" + + ."cd /scratch/statmt/MADAMIRA-release-20140709-1.0 \n"; + print RUN_FILE "java -Xmx2500m -Xms2500m -XX:NewRatio=3 -jar /scratch/statmt/MADAMIRA-release-20140709-1.0/MADAMIRA.jar " + ."-rawinput $splitDir/$file -rawoutdir $out2Dir -rawconfig /scratch/statmt/MADAMIRA-release-20140709-1.0/samples/sampleConfigFile.xml \n"; + + close(RUN_FILE); + + my $cmd = "qsub $qsubFile"; + `$cmd`; + +} + diff --git a/mosesdecoder/contrib/memscore/configure b/mosesdecoder/contrib/memscore/configure new file mode 100644 index 0000000000000000000000000000000000000000..3849f2c29325a3dcbb4dd2ecc358b29c2b461404 --- /dev/null +++ b/mosesdecoder/contrib/memscore/configure @@ -0,0 +1,7194 @@ +#! /bin/sh +# Guess values for system-dependent variables and create Makefiles. +# Generated by GNU Autoconf 2.63 for memscore 1.0. +# +# Report bugs to . +# +# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, +# 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. +# This configure script is free software; the Free Software Foundation +# gives unlimited permission to copy, distribute and modify it. +## --------------------- ## +## M4sh Initialization. ## +## --------------------- ## + +# Be more Bourne compatible +DUALCASE=1; export DUALCASE # for MKS sh +if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then + emulate sh + NULLCMD=: + # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which + # is contrary to our usage. Disable this feature. + alias -g '${1+"$@"}'='"$@"' + setopt NO_GLOB_SUBST +else + case `(set -o) 2>/dev/null` in + *posix*) set -o posix ;; +esac + +fi + + + + +# PATH needs CR +# Avoid depending upon Character Ranges. +as_cr_letters='abcdefghijklmnopqrstuvwxyz' +as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ' +as_cr_Letters=$as_cr_letters$as_cr_LETTERS +as_cr_digits='0123456789' +as_cr_alnum=$as_cr_Letters$as_cr_digits + +as_nl=' +' +export as_nl +# Printing a long string crashes Solaris 7 /usr/bin/printf. +as_echo='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\' +as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo +as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo$as_echo +if (test "X`printf %s $as_echo`" = "X$as_echo") 2>/dev/null; then + as_echo='printf %s\n' + as_echo_n='printf %s' +else + if test "X`(/usr/ucb/echo -n -n $as_echo) 2>/dev/null`" = "X-n $as_echo"; then + as_echo_body='eval /usr/ucb/echo -n "$1$as_nl"' + as_echo_n='/usr/ucb/echo -n' + else + as_echo_body='eval expr "X$1" : "X\\(.*\\)"' + as_echo_n_body='eval + arg=$1; + case $arg in + *"$as_nl"*) + expr "X$arg" : "X\\(.*\\)$as_nl"; + arg=`expr "X$arg" : ".*$as_nl\\(.*\\)"`;; + esac; + expr "X$arg" : "X\\(.*\\)" | tr -d "$as_nl" + ' + export as_echo_n_body + as_echo_n='sh -c $as_echo_n_body as_echo' + fi + export as_echo_body + as_echo='sh -c $as_echo_body as_echo' +fi + +# The user is always right. +if test "${PATH_SEPARATOR+set}" != set; then + PATH_SEPARATOR=: + (PATH='/bin;/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 && { + (PATH='/bin:/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 || + PATH_SEPARATOR=';' + } +fi + +# Support unset when possible. +if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then + as_unset=unset +else + as_unset=false +fi + + +# IFS +# We need space, tab and new line, in precisely that order. Quoting is +# there to prevent editors from complaining about space-tab. +# (If _AS_PATH_WALK were called with IFS unset, it would disable word +# splitting by setting IFS to empty value.) +IFS=" "" $as_nl" + +# Find who we are. Look in the path if we contain no directory separator. +case $0 in + *[\\/]* ) as_myself=$0 ;; + *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break +done +IFS=$as_save_IFS + + ;; +esac +# We did not find ourselves, most probably we were run as `sh COMMAND' +# in which case we are not to be found in the path. +if test "x$as_myself" = x; then + as_myself=$0 +fi +if test ! -f "$as_myself"; then + $as_echo "$as_myself: error: cannot find myself; rerun with an absolute file name" >&2 + { (exit 1); exit 1; } +fi + +# Work around bugs in pre-3.0 UWIN ksh. +for as_var in ENV MAIL MAILPATH +do ($as_unset $as_var) >/dev/null 2>&1 && $as_unset $as_var +done +PS1='$ ' +PS2='> ' +PS4='+ ' + +# NLS nuisances. +LC_ALL=C +export LC_ALL +LANGUAGE=C +export LANGUAGE + +# Required to use basename. +if expr a : '\(a\)' >/dev/null 2>&1 && + test "X`expr 00001 : '.*\(...\)'`" = X001; then + as_expr=expr +else + as_expr=false +fi + +if (basename -- /) >/dev/null 2>&1 && test "X`basename -- / 2>&1`" = "X/"; then + as_basename=basename +else + as_basename=false +fi + + +# Name of the executable. +as_me=`$as_basename -- "$0" || +$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \ + X"$0" : 'X\(//\)$' \| \ + X"$0" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X/"$0" | + sed '/^.*\/\([^/][^/]*\)\/*$/{ + s//\1/ + q + } + /^X\/\(\/\/\)$/{ + s//\1/ + q + } + /^X\/\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'` + +# CDPATH. +$as_unset CDPATH + + +if test "x$CONFIG_SHELL" = x; then + if (eval ":") 2>/dev/null; then + as_have_required=yes +else + as_have_required=no +fi + + if test $as_have_required = yes && (eval ": +(as_func_return () { + (exit \$1) +} +as_func_success () { + as_func_return 0 +} +as_func_failure () { + as_func_return 1 +} +as_func_ret_success () { + return 0 +} +as_func_ret_failure () { + return 1 +} + +exitcode=0 +if as_func_success; then + : +else + exitcode=1 + echo as_func_success failed. +fi + +if as_func_failure; then + exitcode=1 + echo as_func_failure succeeded. +fi + +if as_func_ret_success; then + : +else + exitcode=1 + echo as_func_ret_success failed. +fi + +if as_func_ret_failure; then + exitcode=1 + echo as_func_ret_failure succeeded. +fi + +if ( set x; as_func_ret_success y && test x = \"\$1\" ); then + : +else + exitcode=1 + echo positional parameters were not saved. +fi + +test \$exitcode = 0) || { (exit 1); exit 1; } + +( + as_lineno_1=\$LINENO + as_lineno_2=\$LINENO + test \"x\$as_lineno_1\" != \"x\$as_lineno_2\" && + test \"x\`expr \$as_lineno_1 + 1\`\" = \"x\$as_lineno_2\") || { (exit 1); exit 1; } +") 2> /dev/null; then + : +else + as_candidate_shells= + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + case $as_dir in + /*) + for as_base in sh bash ksh sh5; do + as_candidate_shells="$as_candidate_shells $as_dir/$as_base" + done;; + esac +done +IFS=$as_save_IFS + + + for as_shell in $as_candidate_shells $SHELL; do + # Try only shells that exist, to save several forks. + if { test -f "$as_shell" || test -f "$as_shell.exe"; } && + { ("$as_shell") 2> /dev/null <<\_ASEOF +if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then + emulate sh + NULLCMD=: + # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which + # is contrary to our usage. Disable this feature. + alias -g '${1+"$@"}'='"$@"' + setopt NO_GLOB_SUBST +else + case `(set -o) 2>/dev/null` in + *posix*) set -o posix ;; +esac + +fi + + +: +_ASEOF +}; then + CONFIG_SHELL=$as_shell + as_have_required=yes + if { "$as_shell" 2> /dev/null <<\_ASEOF +if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then + emulate sh + NULLCMD=: + # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which + # is contrary to our usage. Disable this feature. + alias -g '${1+"$@"}'='"$@"' + setopt NO_GLOB_SUBST +else + case `(set -o) 2>/dev/null` in + *posix*) set -o posix ;; +esac + +fi + + +: +(as_func_return () { + (exit $1) +} +as_func_success () { + as_func_return 0 +} +as_func_failure () { + as_func_return 1 +} +as_func_ret_success () { + return 0 +} +as_func_ret_failure () { + return 1 +} + +exitcode=0 +if as_func_success; then + : +else + exitcode=1 + echo as_func_success failed. +fi + +if as_func_failure; then + exitcode=1 + echo as_func_failure succeeded. +fi + +if as_func_ret_success; then + : +else + exitcode=1 + echo as_func_ret_success failed. +fi + +if as_func_ret_failure; then + exitcode=1 + echo as_func_ret_failure succeeded. +fi + +if ( set x; as_func_ret_success y && test x = "$1" ); then + : +else + exitcode=1 + echo positional parameters were not saved. +fi + +test $exitcode = 0) || { (exit 1); exit 1; } + +( + as_lineno_1=$LINENO + as_lineno_2=$LINENO + test "x$as_lineno_1" != "x$as_lineno_2" && + test "x`expr $as_lineno_1 + 1`" = "x$as_lineno_2") || { (exit 1); exit 1; } + +_ASEOF +}; then + break +fi + +fi + + done + + if test "x$CONFIG_SHELL" != x; then + for as_var in BASH_ENV ENV + do ($as_unset $as_var) >/dev/null 2>&1 && $as_unset $as_var + done + export CONFIG_SHELL + exec "$CONFIG_SHELL" "$as_myself" ${1+"$@"} +fi + + + if test $as_have_required = no; then + echo This script requires a shell more modern than all the + echo shells that I found on your system. Please install a + echo modern shell, or manually run the script under such a + echo shell if you do have one. + { (exit 1); exit 1; } +fi + + +fi + +fi + + + +(eval "as_func_return () { + (exit \$1) +} +as_func_success () { + as_func_return 0 +} +as_func_failure () { + as_func_return 1 +} +as_func_ret_success () { + return 0 +} +as_func_ret_failure () { + return 1 +} + +exitcode=0 +if as_func_success; then + : +else + exitcode=1 + echo as_func_success failed. +fi + +if as_func_failure; then + exitcode=1 + echo as_func_failure succeeded. +fi + +if as_func_ret_success; then + : +else + exitcode=1 + echo as_func_ret_success failed. +fi + +if as_func_ret_failure; then + exitcode=1 + echo as_func_ret_failure succeeded. +fi + +if ( set x; as_func_ret_success y && test x = \"\$1\" ); then + : +else + exitcode=1 + echo positional parameters were not saved. +fi + +test \$exitcode = 0") || { + echo No shell found that supports shell functions. + echo Please tell bug-autoconf@gnu.org about your system, + echo including any error possibly output before this message. + echo This can help us improve future autoconf versions. + echo Configuration will now proceed without shell functions. +} + + + + as_lineno_1=$LINENO + as_lineno_2=$LINENO + test "x$as_lineno_1" != "x$as_lineno_2" && + test "x`expr $as_lineno_1 + 1`" = "x$as_lineno_2" || { + + # Create $as_me.lineno as a copy of $as_myself, but with $LINENO + # uniformly replaced by the line number. The first 'sed' inserts a + # line-number line after each line using $LINENO; the second 'sed' + # does the real work. The second script uses 'N' to pair each + # line-number line with the line containing $LINENO, and appends + # trailing '-' during substitution so that $LINENO is not a special + # case at line end. + # (Raja R Harinath suggested sed '=', and Paul Eggert wrote the + # scripts with optimization help from Paolo Bonzini. Blame Lee + # E. McMahon (1931-1989) for sed's syntax. :-) + sed -n ' + p + /[$]LINENO/= + ' <$as_myself | + sed ' + s/[$]LINENO.*/&-/ + t lineno + b + :lineno + N + :loop + s/[$]LINENO\([^'$as_cr_alnum'_].*\n\)\(.*\)/\2\1\2/ + t loop + s/-\n.*// + ' >$as_me.lineno && + chmod +x "$as_me.lineno" || + { $as_echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2 + { (exit 1); exit 1; }; } + + # Don't try to exec as it changes $[0], causing all sort of problems + # (the dirname of $[0] is not the place where we might find the + # original and so on. Autoconf is especially sensitive to this). + . "./$as_me.lineno" + # Exit status is that of the last command. + exit +} + + +if (as_dir=`dirname -- /` && test "X$as_dir" = X/) >/dev/null 2>&1; then + as_dirname=dirname +else + as_dirname=false +fi + +ECHO_C= ECHO_N= ECHO_T= +case `echo -n x` in +-n*) + case `echo 'x\c'` in + *c*) ECHO_T=' ';; # ECHO_T is single tab character. + *) ECHO_C='\c';; + esac;; +*) + ECHO_N='-n';; +esac +if expr a : '\(a\)' >/dev/null 2>&1 && + test "X`expr 00001 : '.*\(...\)'`" = X001; then + as_expr=expr +else + as_expr=false +fi + +rm -f conf$$ conf$$.exe conf$$.file +if test -d conf$$.dir; then + rm -f conf$$.dir/conf$$.file +else + rm -f conf$$.dir + mkdir conf$$.dir 2>/dev/null +fi +if (echo >conf$$.file) 2>/dev/null; then + if ln -s conf$$.file conf$$ 2>/dev/null; then + as_ln_s='ln -s' + # ... but there are two gotchas: + # 1) On MSYS, both `ln -s file dir' and `ln file dir' fail. + # 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable. + # In both cases, we have to default to `cp -p'. + ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe || + as_ln_s='cp -p' + elif ln conf$$.file conf$$ 2>/dev/null; then + as_ln_s=ln + else + as_ln_s='cp -p' + fi +else + as_ln_s='cp -p' +fi +rm -f conf$$ conf$$.exe conf$$.dir/conf$$.file conf$$.file +rmdir conf$$.dir 2>/dev/null + +if mkdir -p . 2>/dev/null; then + as_mkdir_p=: +else + test -d ./-p && rmdir ./-p + as_mkdir_p=false +fi + +if test -x / >/dev/null 2>&1; then + as_test_x='test -x' +else + if ls -dL / >/dev/null 2>&1; then + as_ls_L_option=L + else + as_ls_L_option= + fi + as_test_x=' + eval sh -c '\'' + if test -d "$1"; then + test -d "$1/."; + else + case $1 in + -*)set "./$1";; + esac; + case `ls -ld'$as_ls_L_option' "$1" 2>/dev/null` in + ???[sx]*):;;*)false;;esac;fi + '\'' sh + ' +fi +as_executable_p=$as_test_x + +# Sed expression to map a string onto a valid CPP name. +as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'" + +# Sed expression to map a string onto a valid variable name. +as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'" + + + +exec 7<&0 &1 + +# Name of the host. +# hostname on some systems (SVR3.2, Linux) returns a bogus exit status, +# so uname gets run too. +ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q` + +# +# Initializations. +# +ac_default_prefix=/usr/local +ac_clean_files= +ac_config_libobj_dir=. +LIBOBJS= +cross_compiling=no +subdirs= +MFLAGS= +MAKEFLAGS= +SHELL=${CONFIG_SHELL-/bin/sh} + +# Identity of this package. +PACKAGE_NAME='memscore' +PACKAGE_TARNAME='memscore' +PACKAGE_VERSION='1.0' +PACKAGE_STRING='memscore 1.0' +PACKAGE_BUGREPORT='hardmeier at fbk.eu' + +ac_unique_file="memscore.cpp" +# Factoring default headers for most tests. +ac_includes_default="\ +#include +#ifdef HAVE_SYS_TYPES_H +# include +#endif +#ifdef HAVE_SYS_STAT_H +# include +#endif +#ifdef STDC_HEADERS +# include +# include +#else +# ifdef HAVE_STDLIB_H +# include +# endif +#endif +#ifdef HAVE_STRING_H +# if !defined STDC_HEADERS && defined HAVE_MEMORY_H +# include +# endif +# include +#endif +#ifdef HAVE_STRINGS_H +# include +#endif +#ifdef HAVE_INTTYPES_H +# include +#endif +#ifdef HAVE_STDINT_H +# include +#endif +#ifdef HAVE_UNISTD_H +# include +#endif" + +ac_subst_vars='LTLIBOBJS +LIBOBJS +CHANNEL_SCORER_FALSE +CHANNEL_SCORER_TRUE +IRSTLM_FALSE +IRSTLM_TRUE +EGREP +GREP +CXXCPP +BOOST_LDFLAGS +BOOST_CPPFLAGS +am__fastdepCC_FALSE +am__fastdepCC_TRUE +CCDEPMODE +ac_ct_CC +CFLAGS +CC +am__fastdepCXX_FALSE +am__fastdepCXX_TRUE +CXXDEPMODE +AMDEPBACKSLASH +AMDEP_FALSE +AMDEP_TRUE +am__quote +am__include +DEPDIR +OBJEXT +EXEEXT +ac_ct_CXX +CPPFLAGS +LDFLAGS +CXXFLAGS +CXX +am__untar +am__tar +AMTAR +am__leading_dot +SET_MAKE +AWK +mkdir_p +INSTALL_STRIP_PROGRAM +STRIP +install_sh +MAKEINFO +AUTOHEADER +AUTOMAKE +AUTOCONF +ACLOCAL +VERSION +PACKAGE +CYGPATH_W +INSTALL_DATA +INSTALL_SCRIPT +INSTALL_PROGRAM +target_alias +host_alias +build_alias +LIBS +ECHO_T +ECHO_N +ECHO_C +DEFS +mandir +localedir +libdir +psdir +pdfdir +dvidir +htmldir +infodir +docdir +oldincludedir +includedir +localstatedir +sharedstatedir +sysconfdir +datadir +datarootdir +libexecdir +sbindir +bindir +program_transform_name +prefix +exec_prefix +PACKAGE_BUGREPORT +PACKAGE_STRING +PACKAGE_VERSION +PACKAGE_TARNAME +PACKAGE_NAME +PATH_SEPARATOR +SHELL' +ac_subst_files='' +ac_user_opts=' +enable_option_checking +with_irstlm +with_gsl +enable_channel +enable_dependency_tracking +with_boost +with_boost_libdir +' + ac_precious_vars='build_alias +host_alias +target_alias +CXX +CXXFLAGS +LDFLAGS +LIBS +CPPFLAGS +CCC +CC +CFLAGS +CXXCPP' + + +# Initialize some variables set by options. +ac_init_help= +ac_init_version=false +ac_unrecognized_opts= +ac_unrecognized_sep= +# The variables have the same names as the options, with +# dashes changed to underlines. +cache_file=/dev/null +exec_prefix=NONE +no_create= +no_recursion= +prefix=NONE +program_prefix=NONE +program_suffix=NONE +program_transform_name=s,x,x, +silent= +site= +srcdir= +verbose= +x_includes=NONE +x_libraries=NONE + +# Installation directory options. +# These are left unexpanded so users can "make install exec_prefix=/foo" +# and all the variables that are supposed to be based on exec_prefix +# by default will actually change. +# Use braces instead of parens because sh, perl, etc. also accept them. +# (The list follows the same order as the GNU Coding Standards.) +bindir='${exec_prefix}/bin' +sbindir='${exec_prefix}/sbin' +libexecdir='${exec_prefix}/libexec' +datarootdir='${prefix}/share' +datadir='${datarootdir}' +sysconfdir='${prefix}/etc' +sharedstatedir='${prefix}/com' +localstatedir='${prefix}/var' +includedir='${prefix}/include' +oldincludedir='/usr/include' +docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' +infodir='${datarootdir}/info' +htmldir='${docdir}' +dvidir='${docdir}' +pdfdir='${docdir}' +psdir='${docdir}' +libdir='${exec_prefix}/lib' +localedir='${datarootdir}/locale' +mandir='${datarootdir}/man' + +ac_prev= +ac_dashdash= +for ac_option +do + # If the previous option needs an argument, assign it. + if test -n "$ac_prev"; then + eval $ac_prev=\$ac_option + ac_prev= + continue + fi + + case $ac_option in + *=*) ac_optarg=`expr "X$ac_option" : '[^=]*=\(.*\)'` ;; + *) ac_optarg=yes ;; + esac + + # Accept the important Cygnus configure options, so we can diagnose typos. + + case $ac_dashdash$ac_option in + --) + ac_dashdash=yes ;; + + -bindir | --bindir | --bindi | --bind | --bin | --bi) + ac_prev=bindir ;; + -bindir=* | --bindir=* | --bindi=* | --bind=* | --bin=* | --bi=*) + bindir=$ac_optarg ;; + + -build | --build | --buil | --bui | --bu) + ac_prev=build_alias ;; + -build=* | --build=* | --buil=* | --bui=* | --bu=*) + build_alias=$ac_optarg ;; + + -cache-file | --cache-file | --cache-fil | --cache-fi \ + | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c) + ac_prev=cache_file ;; + -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \ + | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* | --c=*) + cache_file=$ac_optarg ;; + + --config-cache | -C) + cache_file=config.cache ;; + + -datadir | --datadir | --datadi | --datad) + ac_prev=datadir ;; + -datadir=* | --datadir=* | --datadi=* | --datad=*) + datadir=$ac_optarg ;; + + -datarootdir | --datarootdir | --datarootdi | --datarootd | --dataroot \ + | --dataroo | --dataro | --datar) + ac_prev=datarootdir ;; + -datarootdir=* | --datarootdir=* | --datarootdi=* | --datarootd=* \ + | --dataroot=* | --dataroo=* | --dataro=* | --datar=*) + datarootdir=$ac_optarg ;; + + -disable-* | --disable-*) + ac_useropt=`expr "x$ac_option" : 'x-*disable-\(.*\)'` + # Reject names that are not valid shell variable names. + expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && + { $as_echo "$as_me: error: invalid feature name: $ac_useropt" >&2 + { (exit 1); exit 1; }; } + ac_useropt_orig=$ac_useropt + ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` + case $ac_user_opts in + *" +"enable_$ac_useropt" +"*) ;; + *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--disable-$ac_useropt_orig" + ac_unrecognized_sep=', ';; + esac + eval enable_$ac_useropt=no ;; + + -docdir | --docdir | --docdi | --doc | --do) + ac_prev=docdir ;; + -docdir=* | --docdir=* | --docdi=* | --doc=* | --do=*) + docdir=$ac_optarg ;; + + -dvidir | --dvidir | --dvidi | --dvid | --dvi | --dv) + ac_prev=dvidir ;; + -dvidir=* | --dvidir=* | --dvidi=* | --dvid=* | --dvi=* | --dv=*) + dvidir=$ac_optarg ;; + + -enable-* | --enable-*) + ac_useropt=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'` + # Reject names that are not valid shell variable names. + expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && + { $as_echo "$as_me: error: invalid feature name: $ac_useropt" >&2 + { (exit 1); exit 1; }; } + ac_useropt_orig=$ac_useropt + ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` + case $ac_user_opts in + *" +"enable_$ac_useropt" +"*) ;; + *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--enable-$ac_useropt_orig" + ac_unrecognized_sep=', ';; + esac + eval enable_$ac_useropt=\$ac_optarg ;; + + -exec-prefix | --exec_prefix | --exec-prefix | --exec-prefi \ + | --exec-pref | --exec-pre | --exec-pr | --exec-p | --exec- \ + | --exec | --exe | --ex) + ac_prev=exec_prefix ;; + -exec-prefix=* | --exec_prefix=* | --exec-prefix=* | --exec-prefi=* \ + | --exec-pref=* | --exec-pre=* | --exec-pr=* | --exec-p=* | --exec-=* \ + | --exec=* | --exe=* | --ex=*) + exec_prefix=$ac_optarg ;; + + -gas | --gas | --ga | --g) + # Obsolete; use --with-gas. + with_gas=yes ;; + + -help | --help | --hel | --he | -h) + ac_init_help=long ;; + -help=r* | --help=r* | --hel=r* | --he=r* | -hr*) + ac_init_help=recursive ;; + -help=s* | --help=s* | --hel=s* | --he=s* | -hs*) + ac_init_help=short ;; + + -host | --host | --hos | --ho) + ac_prev=host_alias ;; + -host=* | --host=* | --hos=* | --ho=*) + host_alias=$ac_optarg ;; + + -htmldir | --htmldir | --htmldi | --htmld | --html | --htm | --ht) + ac_prev=htmldir ;; + -htmldir=* | --htmldir=* | --htmldi=* | --htmld=* | --html=* | --htm=* \ + | --ht=*) + htmldir=$ac_optarg ;; + + -includedir | --includedir | --includedi | --included | --include \ + | --includ | --inclu | --incl | --inc) + ac_prev=includedir ;; + -includedir=* | --includedir=* | --includedi=* | --included=* | --include=* \ + | --includ=* | --inclu=* | --incl=* | --inc=*) + includedir=$ac_optarg ;; + + -infodir | --infodir | --infodi | --infod | --info | --inf) + ac_prev=infodir ;; + -infodir=* | --infodir=* | --infodi=* | --infod=* | --info=* | --inf=*) + infodir=$ac_optarg ;; + + -libdir | --libdir | --libdi | --libd) + ac_prev=libdir ;; + -libdir=* | --libdir=* | --libdi=* | --libd=*) + libdir=$ac_optarg ;; + + -libexecdir | --libexecdir | --libexecdi | --libexecd | --libexec \ + | --libexe | --libex | --libe) + ac_prev=libexecdir ;; + -libexecdir=* | --libexecdir=* | --libexecdi=* | --libexecd=* | --libexec=* \ + | --libexe=* | --libex=* | --libe=*) + libexecdir=$ac_optarg ;; + + -localedir | --localedir | --localedi | --localed | --locale) + ac_prev=localedir ;; + -localedir=* | --localedir=* | --localedi=* | --localed=* | --locale=*) + localedir=$ac_optarg ;; + + -localstatedir | --localstatedir | --localstatedi | --localstated \ + | --localstate | --localstat | --localsta | --localst | --locals) + ac_prev=localstatedir ;; + -localstatedir=* | --localstatedir=* | --localstatedi=* | --localstated=* \ + | --localstate=* | --localstat=* | --localsta=* | --localst=* | --locals=*) + localstatedir=$ac_optarg ;; + + -mandir | --mandir | --mandi | --mand | --man | --ma | --m) + ac_prev=mandir ;; + -mandir=* | --mandir=* | --mandi=* | --mand=* | --man=* | --ma=* | --m=*) + mandir=$ac_optarg ;; + + -nfp | --nfp | --nf) + # Obsolete; use --without-fp. + with_fp=no ;; + + -no-create | --no-create | --no-creat | --no-crea | --no-cre \ + | --no-cr | --no-c | -n) + no_create=yes ;; + + -no-recursion | --no-recursion | --no-recursio | --no-recursi \ + | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r) + no_recursion=yes ;; + + -oldincludedir | --oldincludedir | --oldincludedi | --oldincluded \ + | --oldinclude | --oldinclud | --oldinclu | --oldincl | --oldinc \ + | --oldin | --oldi | --old | --ol | --o) + ac_prev=oldincludedir ;; + -oldincludedir=* | --oldincludedir=* | --oldincludedi=* | --oldincluded=* \ + | --oldinclude=* | --oldinclud=* | --oldinclu=* | --oldincl=* | --oldinc=* \ + | --oldin=* | --oldi=* | --old=* | --ol=* | --o=*) + oldincludedir=$ac_optarg ;; + + -prefix | --prefix | --prefi | --pref | --pre | --pr | --p) + ac_prev=prefix ;; + -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*) + prefix=$ac_optarg ;; + + -program-prefix | --program-prefix | --program-prefi | --program-pref \ + | --program-pre | --program-pr | --program-p) + ac_prev=program_prefix ;; + -program-prefix=* | --program-prefix=* | --program-prefi=* \ + | --program-pref=* | --program-pre=* | --program-pr=* | --program-p=*) + program_prefix=$ac_optarg ;; + + -program-suffix | --program-suffix | --program-suffi | --program-suff \ + | --program-suf | --program-su | --program-s) + ac_prev=program_suffix ;; + -program-suffix=* | --program-suffix=* | --program-suffi=* \ + | --program-suff=* | --program-suf=* | --program-su=* | --program-s=*) + program_suffix=$ac_optarg ;; + + -program-transform-name | --program-transform-name \ + | --program-transform-nam | --program-transform-na \ + | --program-transform-n | --program-transform- \ + | --program-transform | --program-transfor \ + | --program-transfo | --program-transf \ + | --program-trans | --program-tran \ + | --progr-tra | --program-tr | --program-t) + ac_prev=program_transform_name ;; + -program-transform-name=* | --program-transform-name=* \ + | --program-transform-nam=* | --program-transform-na=* \ + | --program-transform-n=* | --program-transform-=* \ + | --program-transform=* | --program-transfor=* \ + | --program-transfo=* | --program-transf=* \ + | --program-trans=* | --program-tran=* \ + | --progr-tra=* | --program-tr=* | --program-t=*) + program_transform_name=$ac_optarg ;; + + -pdfdir | --pdfdir | --pdfdi | --pdfd | --pdf | --pd) + ac_prev=pdfdir ;; + -pdfdir=* | --pdfdir=* | --pdfdi=* | --pdfd=* | --pdf=* | --pd=*) + pdfdir=$ac_optarg ;; + + -psdir | --psdir | --psdi | --psd | --ps) + ac_prev=psdir ;; + -psdir=* | --psdir=* | --psdi=* | --psd=* | --ps=*) + psdir=$ac_optarg ;; + + -q | -quiet | --quiet | --quie | --qui | --qu | --q \ + | -silent | --silent | --silen | --sile | --sil) + silent=yes ;; + + -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) + ac_prev=sbindir ;; + -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ + | --sbi=* | --sb=*) + sbindir=$ac_optarg ;; + + -sharedstatedir | --sharedstatedir | --sharedstatedi \ + | --sharedstated | --sharedstate | --sharedstat | --sharedsta \ + | --sharedst | --shareds | --shared | --share | --shar \ + | --sha | --sh) + ac_prev=sharedstatedir ;; + -sharedstatedir=* | --sharedstatedir=* | --sharedstatedi=* \ + | --sharedstated=* | --sharedstate=* | --sharedstat=* | --sharedsta=* \ + | --sharedst=* | --shareds=* | --shared=* | --share=* | --shar=* \ + | --sha=* | --sh=*) + sharedstatedir=$ac_optarg ;; + + -site | --site | --sit) + ac_prev=site ;; + -site=* | --site=* | --sit=*) + site=$ac_optarg ;; + + -srcdir | --srcdir | --srcdi | --srcd | --src | --sr) + ac_prev=srcdir ;; + -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*) + srcdir=$ac_optarg ;; + + -sysconfdir | --sysconfdir | --sysconfdi | --sysconfd | --sysconf \ + | --syscon | --sysco | --sysc | --sys | --sy) + ac_prev=sysconfdir ;; + -sysconfdir=* | --sysconfdir=* | --sysconfdi=* | --sysconfd=* | --sysconf=* \ + | --syscon=* | --sysco=* | --sysc=* | --sys=* | --sy=*) + sysconfdir=$ac_optarg ;; + + -target | --target | --targe | --targ | --tar | --ta | --t) + ac_prev=target_alias ;; + -target=* | --target=* | --targe=* | --targ=* | --tar=* | --ta=* | --t=*) + target_alias=$ac_optarg ;; + + -v | -verbose | --verbose | --verbos | --verbo | --verb) + verbose=yes ;; + + -version | --version | --versio | --versi | --vers | -V) + ac_init_version=: ;; + + -with-* | --with-*) + ac_useropt=`expr "x$ac_option" : 'x-*with-\([^=]*\)'` + # Reject names that are not valid shell variable names. + expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && + { $as_echo "$as_me: error: invalid package name: $ac_useropt" >&2 + { (exit 1); exit 1; }; } + ac_useropt_orig=$ac_useropt + ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` + case $ac_user_opts in + *" +"with_$ac_useropt" +"*) ;; + *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--with-$ac_useropt_orig" + ac_unrecognized_sep=', ';; + esac + eval with_$ac_useropt=\$ac_optarg ;; + + -without-* | --without-*) + ac_useropt=`expr "x$ac_option" : 'x-*without-\(.*\)'` + # Reject names that are not valid shell variable names. + expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && + { $as_echo "$as_me: error: invalid package name: $ac_useropt" >&2 + { (exit 1); exit 1; }; } + ac_useropt_orig=$ac_useropt + ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` + case $ac_user_opts in + *" +"with_$ac_useropt" +"*) ;; + *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--without-$ac_useropt_orig" + ac_unrecognized_sep=', ';; + esac + eval with_$ac_useropt=no ;; + + --x) + # Obsolete; use --with-x. + with_x=yes ;; + + -x-includes | --x-includes | --x-include | --x-includ | --x-inclu \ + | --x-incl | --x-inc | --x-in | --x-i) + ac_prev=x_includes ;; + -x-includes=* | --x-includes=* | --x-include=* | --x-includ=* | --x-inclu=* \ + | --x-incl=* | --x-inc=* | --x-in=* | --x-i=*) + x_includes=$ac_optarg ;; + + -x-libraries | --x-libraries | --x-librarie | --x-librari \ + | --x-librar | --x-libra | --x-libr | --x-lib | --x-li | --x-l) + ac_prev=x_libraries ;; + -x-libraries=* | --x-libraries=* | --x-librarie=* | --x-librari=* \ + | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*) + x_libraries=$ac_optarg ;; + + -*) { $as_echo "$as_me: error: unrecognized option: $ac_option +Try \`$0 --help' for more information." >&2 + { (exit 1); exit 1; }; } + ;; + + *=*) + ac_envvar=`expr "x$ac_option" : 'x\([^=]*\)='` + # Reject names that are not valid shell variable names. + expr "x$ac_envvar" : ".*[^_$as_cr_alnum]" >/dev/null && + { $as_echo "$as_me: error: invalid variable name: $ac_envvar" >&2 + { (exit 1); exit 1; }; } + eval $ac_envvar=\$ac_optarg + export $ac_envvar ;; + + *) + # FIXME: should be removed in autoconf 3.0. + $as_echo "$as_me: WARNING: you should use --build, --host, --target" >&2 + expr "x$ac_option" : ".*[^-._$as_cr_alnum]" >/dev/null && + $as_echo "$as_me: WARNING: invalid host type: $ac_option" >&2 + : ${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option} + ;; + + esac +done + +if test -n "$ac_prev"; then + ac_option=--`echo $ac_prev | sed 's/_/-/g'` + { $as_echo "$as_me: error: missing argument to $ac_option" >&2 + { (exit 1); exit 1; }; } +fi + +if test -n "$ac_unrecognized_opts"; then + case $enable_option_checking in + no) ;; + fatal) { $as_echo "$as_me: error: unrecognized options: $ac_unrecognized_opts" >&2 + { (exit 1); exit 1; }; } ;; + *) $as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2 ;; + esac +fi + +# Check all directory arguments for consistency. +for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ + datadir sysconfdir sharedstatedir localstatedir includedir \ + oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ + libdir localedir mandir +do + eval ac_val=\$$ac_var + # Remove trailing slashes. + case $ac_val in + */ ) + ac_val=`expr "X$ac_val" : 'X\(.*[^/]\)' \| "X$ac_val" : 'X\(.*\)'` + eval $ac_var=\$ac_val;; + esac + # Be sure to have absolute directory names. + case $ac_val in + [\\/$]* | ?:[\\/]* ) continue;; + NONE | '' ) case $ac_var in *prefix ) continue;; esac;; + esac + { $as_echo "$as_me: error: expected an absolute directory name for --$ac_var: $ac_val" >&2 + { (exit 1); exit 1; }; } +done + +# There might be people who depend on the old broken behavior: `$host' +# used to hold the argument of --host etc. +# FIXME: To remove some day. +build=$build_alias +host=$host_alias +target=$target_alias + +# FIXME: To remove some day. +if test "x$host_alias" != x; then + if test "x$build_alias" = x; then + cross_compiling=maybe + $as_echo "$as_me: WARNING: If you wanted to set the --build type, don't use --host. + If a cross compiler is detected then cross compile mode will be used." >&2 + elif test "x$build_alias" != "x$host_alias"; then + cross_compiling=yes + fi +fi + +ac_tool_prefix= +test -n "$host_alias" && ac_tool_prefix=$host_alias- + +test "$silent" = yes && exec 6>/dev/null + + +ac_pwd=`pwd` && test -n "$ac_pwd" && +ac_ls_di=`ls -di .` && +ac_pwd_ls_di=`cd "$ac_pwd" && ls -di .` || + { $as_echo "$as_me: error: working directory cannot be determined" >&2 + { (exit 1); exit 1; }; } +test "X$ac_ls_di" = "X$ac_pwd_ls_di" || + { $as_echo "$as_me: error: pwd does not report name of working directory" >&2 + { (exit 1); exit 1; }; } + + +# Find the source files, if location was not specified. +if test -z "$srcdir"; then + ac_srcdir_defaulted=yes + # Try the directory containing this script, then the parent directory. + ac_confdir=`$as_dirname -- "$as_myself" || +$as_expr X"$as_myself" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$as_myself" : 'X\(//\)[^/]' \| \ + X"$as_myself" : 'X\(//\)$' \| \ + X"$as_myself" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X"$as_myself" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'` + srcdir=$ac_confdir + if test ! -r "$srcdir/$ac_unique_file"; then + srcdir=.. + fi +else + ac_srcdir_defaulted=no +fi +if test ! -r "$srcdir/$ac_unique_file"; then + test "$ac_srcdir_defaulted" = yes && srcdir="$ac_confdir or .." + { $as_echo "$as_me: error: cannot find sources ($ac_unique_file) in $srcdir" >&2 + { (exit 1); exit 1; }; } +fi +ac_msg="sources are in $srcdir, but \`cd $srcdir' does not work" +ac_abs_confdir=`( + cd "$srcdir" && test -r "./$ac_unique_file" || { $as_echo "$as_me: error: $ac_msg" >&2 + { (exit 1); exit 1; }; } + pwd)` +# When building in place, set srcdir=. +if test "$ac_abs_confdir" = "$ac_pwd"; then + srcdir=. +fi +# Remove unnecessary trailing slashes from srcdir. +# Double slashes in file names in object file debugging info +# mess up M-x gdb in Emacs. +case $srcdir in +*/) srcdir=`expr "X$srcdir" : 'X\(.*[^/]\)' \| "X$srcdir" : 'X\(.*\)'`;; +esac +for ac_var in $ac_precious_vars; do + eval ac_env_${ac_var}_set=\${${ac_var}+set} + eval ac_env_${ac_var}_value=\$${ac_var} + eval ac_cv_env_${ac_var}_set=\${${ac_var}+set} + eval ac_cv_env_${ac_var}_value=\$${ac_var} +done + +# +# Report the --help message. +# +if test "$ac_init_help" = "long"; then + # Omit some internal or obsolete options to make the list less imposing. + # This message is too long to be a string in the A/UX 3.1 sh. + cat <<_ACEOF +\`configure' configures memscore 1.0 to adapt to many kinds of systems. + +Usage: $0 [OPTION]... [VAR=VALUE]... + +To assign environment variables (e.g., CC, CFLAGS...), specify them as +VAR=VALUE. See below for descriptions of some of the useful variables. + +Defaults for the options are specified in brackets. + +Configuration: + -h, --help display this help and exit + --help=short display options specific to this package + --help=recursive display the short help of all the included packages + -V, --version display version information and exit + -q, --quiet, --silent do not print \`checking...' messages + --cache-file=FILE cache test results in FILE [disabled] + -C, --config-cache alias for \`--cache-file=config.cache' + -n, --no-create do not create output files + --srcdir=DIR find the sources in DIR [configure dir or \`..'] + +Installation directories: + --prefix=PREFIX install architecture-independent files in PREFIX + [$ac_default_prefix] + --exec-prefix=EPREFIX install architecture-dependent files in EPREFIX + [PREFIX] + +By default, \`make install' will install all the files in +\`$ac_default_prefix/bin', \`$ac_default_prefix/lib' etc. You can specify +an installation prefix other than \`$ac_default_prefix' using \`--prefix', +for instance \`--prefix=\$HOME'. + +For better control, use the options below. + +Fine tuning of the installation directories: + --bindir=DIR user executables [EPREFIX/bin] + --sbindir=DIR system admin executables [EPREFIX/sbin] + --libexecdir=DIR program executables [EPREFIX/libexec] + --sysconfdir=DIR read-only single-machine data [PREFIX/etc] + --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] + --localstatedir=DIR modifiable single-machine data [PREFIX/var] + --libdir=DIR object code libraries [EPREFIX/lib] + --includedir=DIR C header files [PREFIX/include] + --oldincludedir=DIR C header files for non-gcc [/usr/include] + --datarootdir=DIR read-only arch.-independent data root [PREFIX/share] + --datadir=DIR read-only architecture-independent data [DATAROOTDIR] + --infodir=DIR info documentation [DATAROOTDIR/info] + --localedir=DIR locale-dependent data [DATAROOTDIR/locale] + --mandir=DIR man documentation [DATAROOTDIR/man] + --docdir=DIR documentation root [DATAROOTDIR/doc/memscore] + --htmldir=DIR html documentation [DOCDIR] + --dvidir=DIR dvi documentation [DOCDIR] + --pdfdir=DIR pdf documentation [DOCDIR] + --psdir=DIR ps documentation [DOCDIR] +_ACEOF + + cat <<\_ACEOF + +Program names: + --program-prefix=PREFIX prepend PREFIX to installed program names + --program-suffix=SUFFIX append SUFFIX to installed program names + --program-transform-name=PROGRAM run sed PROGRAM on installed program names +_ACEOF +fi + +if test -n "$ac_init_help"; then + case $ac_init_help in + short | recursive ) echo "Configuration of memscore 1.0:";; + esac + cat <<\_ACEOF + +Optional Features: + --disable-option-checking ignore unrecognized --enable/--with options + --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) + --enable-FEATURE[=ARG] include FEATURE [ARG=yes] + --enable-channel feature not yet publicly available + --disable-dependency-tracking speeds up one-time build + --enable-dependency-tracking do not reject slow dependency extractors + +Optional Packages: + --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] + --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) + --with-irstlm=PATH (optional) path to the IRSTLM toolkit + --with-gsl=PATH path to the GSL library + --with-boost[=DIR] use boost (default is yes) - it is possible to + specify the root directory for boost (optional) + --with-boost-libdir=LIB_DIR + Force given directory for boost libraries. Note that + this will overwrite library path detection, so use + this parameter only if default library detection + fails and you know exactly where your boost + libraries are located. + +Some influential environment variables: + CXX C++ compiler command + CXXFLAGS C++ compiler flags + LDFLAGS linker flags, e.g. -L if you have libraries in a + nonstandard directory + LIBS libraries to pass to the linker, e.g. -l + CPPFLAGS C/C++/Objective C preprocessor flags, e.g. -I if + you have headers in a nonstandard directory + CC C compiler command + CFLAGS C compiler flags + CXXCPP C++ preprocessor + +Use these variables to override the choices made by `configure' or to help +it to find libraries and programs with nonstandard names/locations. + +Report bugs to . +_ACEOF +ac_status=$? +fi + +if test "$ac_init_help" = "recursive"; then + # If there are subdirs, report their specific --help. + for ac_dir in : $ac_subdirs_all; do test "x$ac_dir" = x: && continue + test -d "$ac_dir" || + { cd "$srcdir" && ac_pwd=`pwd` && srcdir=. && test -d "$ac_dir"; } || + continue + ac_builddir=. + +case "$ac_dir" in +.) ac_dir_suffix= ac_top_builddir_sub=. ac_top_build_prefix= ;; +*) + ac_dir_suffix=/`$as_echo "$ac_dir" | sed 's|^\.[\\/]||'` + # A ".." for each directory in $ac_dir_suffix. + ac_top_builddir_sub=`$as_echo "$ac_dir_suffix" | sed 's|/[^\\/]*|/..|g;s|/||'` + case $ac_top_builddir_sub in + "") ac_top_builddir_sub=. ac_top_build_prefix= ;; + *) ac_top_build_prefix=$ac_top_builddir_sub/ ;; + esac ;; +esac +ac_abs_top_builddir=$ac_pwd +ac_abs_builddir=$ac_pwd$ac_dir_suffix +# for backward compatibility: +ac_top_builddir=$ac_top_build_prefix + +case $srcdir in + .) # We are building in place. + ac_srcdir=. + ac_top_srcdir=$ac_top_builddir_sub + ac_abs_top_srcdir=$ac_pwd ;; + [\\/]* | ?:[\\/]* ) # Absolute name. + ac_srcdir=$srcdir$ac_dir_suffix; + ac_top_srcdir=$srcdir + ac_abs_top_srcdir=$srcdir ;; + *) # Relative name. + ac_srcdir=$ac_top_build_prefix$srcdir$ac_dir_suffix + ac_top_srcdir=$ac_top_build_prefix$srcdir + ac_abs_top_srcdir=$ac_pwd/$srcdir ;; +esac +ac_abs_srcdir=$ac_abs_top_srcdir$ac_dir_suffix + + cd "$ac_dir" || { ac_status=$?; continue; } + # Check for guested configure. + if test -f "$ac_srcdir/configure.gnu"; then + echo && + $SHELL "$ac_srcdir/configure.gnu" --help=recursive + elif test -f "$ac_srcdir/configure"; then + echo && + $SHELL "$ac_srcdir/configure" --help=recursive + else + $as_echo "$as_me: WARNING: no configuration information is in $ac_dir" >&2 + fi || ac_status=$? + cd "$ac_pwd" || { ac_status=$?; break; } + done +fi + +test -n "$ac_init_help" && exit $ac_status +if $ac_init_version; then + cat <<\_ACEOF +memscore configure 1.0 +generated by GNU Autoconf 2.63 + +Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, +2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. +This configure script is free software; the Free Software Foundation +gives unlimited permission to copy, distribute and modify it. +_ACEOF + exit +fi +cat >config.log <<_ACEOF +This file contains any messages produced by compilers while +running configure, to aid debugging if configure makes a mistake. + +It was created by memscore $as_me 1.0, which was +generated by GNU Autoconf 2.63. Invocation command line was + + $ $0 $@ + +_ACEOF +exec 5>>config.log +{ +cat <<_ASUNAME +## --------- ## +## Platform. ## +## --------- ## + +hostname = `(hostname || uname -n) 2>/dev/null | sed 1q` +uname -m = `(uname -m) 2>/dev/null || echo unknown` +uname -r = `(uname -r) 2>/dev/null || echo unknown` +uname -s = `(uname -s) 2>/dev/null || echo unknown` +uname -v = `(uname -v) 2>/dev/null || echo unknown` + +/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null || echo unknown` +/bin/uname -X = `(/bin/uname -X) 2>/dev/null || echo unknown` + +/bin/arch = `(/bin/arch) 2>/dev/null || echo unknown` +/usr/bin/arch -k = `(/usr/bin/arch -k) 2>/dev/null || echo unknown` +/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null || echo unknown` +/usr/bin/hostinfo = `(/usr/bin/hostinfo) 2>/dev/null || echo unknown` +/bin/machine = `(/bin/machine) 2>/dev/null || echo unknown` +/usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null || echo unknown` +/bin/universe = `(/bin/universe) 2>/dev/null || echo unknown` + +_ASUNAME + +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + $as_echo "PATH: $as_dir" +done +IFS=$as_save_IFS + +} >&5 + +cat >&5 <<_ACEOF + + +## ----------- ## +## Core tests. ## +## ----------- ## + +_ACEOF + + +# Keep a trace of the command line. +# Strip out --no-create and --no-recursion so they do not pile up. +# Strip out --silent because we don't want to record it for future runs. +# Also quote any args containing shell meta-characters. +# Make two passes to allow for proper duplicate-argument suppression. +ac_configure_args= +ac_configure_args0= +ac_configure_args1= +ac_must_keep_next=false +for ac_pass in 1 2 +do + for ac_arg + do + case $ac_arg in + -no-create | --no-c* | -n | -no-recursion | --no-r*) continue ;; + -q | -quiet | --quiet | --quie | --qui | --qu | --q \ + | -silent | --silent | --silen | --sile | --sil) + continue ;; + *\'*) + ac_arg=`$as_echo "$ac_arg" | sed "s/'/'\\\\\\\\''/g"` ;; + esac + case $ac_pass in + 1) ac_configure_args0="$ac_configure_args0 '$ac_arg'" ;; + 2) + ac_configure_args1="$ac_configure_args1 '$ac_arg'" + if test $ac_must_keep_next = true; then + ac_must_keep_next=false # Got value, back to normal. + else + case $ac_arg in + *=* | --config-cache | -C | -disable-* | --disable-* \ + | -enable-* | --enable-* | -gas | --g* | -nfp | --nf* \ + | -q | -quiet | --q* | -silent | --sil* | -v | -verb* \ + | -with-* | --with-* | -without-* | --without-* | --x) + case "$ac_configure_args0 " in + "$ac_configure_args1"*" '$ac_arg' "* ) continue ;; + esac + ;; + -* ) ac_must_keep_next=true ;; + esac + fi + ac_configure_args="$ac_configure_args '$ac_arg'" + ;; + esac + done +done +$as_unset ac_configure_args0 || test "${ac_configure_args0+set}" != set || { ac_configure_args0=; export ac_configure_args0; } +$as_unset ac_configure_args1 || test "${ac_configure_args1+set}" != set || { ac_configure_args1=; export ac_configure_args1; } + +# When interrupted or exit'd, cleanup temporary files, and complete +# config.log. We remove comments because anyway the quotes in there +# would cause problems or look ugly. +# WARNING: Use '\'' to represent an apostrophe within the trap. +# WARNING: Do not start the trap code with a newline, due to a FreeBSD 4.0 bug. +trap 'exit_status=$? + # Save into config.log some information that might help in debugging. + { + echo + + cat <<\_ASBOX +## ---------------- ## +## Cache variables. ## +## ---------------- ## +_ASBOX + echo + # The following way of writing the cache mishandles newlines in values, +( + for ac_var in `(set) 2>&1 | sed -n '\''s/^\([a-zA-Z_][a-zA-Z0-9_]*\)=.*/\1/p'\''`; do + eval ac_val=\$$ac_var + case $ac_val in #( + *${as_nl}*) + case $ac_var in #( + *_cv_*) { $as_echo "$as_me:$LINENO: WARNING: cache variable $ac_var contains a newline" >&5 +$as_echo "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;; + esac + case $ac_var in #( + _ | IFS | as_nl) ;; #( + BASH_ARGV | BASH_SOURCE) eval $ac_var= ;; #( + *) $as_unset $ac_var ;; + esac ;; + esac + done + (set) 2>&1 | + case $as_nl`(ac_space='\'' '\''; set) 2>&1` in #( + *${as_nl}ac_space=\ *) + sed -n \ + "s/'\''/'\''\\\\'\'''\''/g; + s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\''\\2'\''/p" + ;; #( + *) + sed -n "/^[_$as_cr_alnum]*_cv_[_$as_cr_alnum]*=/p" + ;; + esac | + sort +) + echo + + cat <<\_ASBOX +## ----------------- ## +## Output variables. ## +## ----------------- ## +_ASBOX + echo + for ac_var in $ac_subst_vars + do + eval ac_val=\$$ac_var + case $ac_val in + *\'\''*) ac_val=`$as_echo "$ac_val" | sed "s/'\''/'\''\\\\\\\\'\'''\''/g"`;; + esac + $as_echo "$ac_var='\''$ac_val'\''" + done | sort + echo + + if test -n "$ac_subst_files"; then + cat <<\_ASBOX +## ------------------- ## +## File substitutions. ## +## ------------------- ## +_ASBOX + echo + for ac_var in $ac_subst_files + do + eval ac_val=\$$ac_var + case $ac_val in + *\'\''*) ac_val=`$as_echo "$ac_val" | sed "s/'\''/'\''\\\\\\\\'\'''\''/g"`;; + esac + $as_echo "$ac_var='\''$ac_val'\''" + done | sort + echo + fi + + if test -s confdefs.h; then + cat <<\_ASBOX +## ----------- ## +## confdefs.h. ## +## ----------- ## +_ASBOX + echo + cat confdefs.h + echo + fi + test "$ac_signal" != 0 && + $as_echo "$as_me: caught signal $ac_signal" + $as_echo "$as_me: exit $exit_status" + } >&5 + rm -f core *.core core.conftest.* && + rm -f -r conftest* confdefs* conf$$* $ac_clean_files && + exit $exit_status +' 0 +for ac_signal in 1 2 13 15; do + trap 'ac_signal='$ac_signal'; { (exit 1); exit 1; }' $ac_signal +done +ac_signal=0 + +# confdefs.h avoids OS command line length limits that DEFS can exceed. +rm -f -r conftest* confdefs.h + +# Predefined preprocessor variables. + +cat >>confdefs.h <<_ACEOF +#define PACKAGE_NAME "$PACKAGE_NAME" +_ACEOF + + +cat >>confdefs.h <<_ACEOF +#define PACKAGE_TARNAME "$PACKAGE_TARNAME" +_ACEOF + + +cat >>confdefs.h <<_ACEOF +#define PACKAGE_VERSION "$PACKAGE_VERSION" +_ACEOF + + +cat >>confdefs.h <<_ACEOF +#define PACKAGE_STRING "$PACKAGE_STRING" +_ACEOF + + +cat >>confdefs.h <<_ACEOF +#define PACKAGE_BUGREPORT "$PACKAGE_BUGREPORT" +_ACEOF + + +# Let the site file select an alternate cache file if it wants to. +# Prefer an explicitly selected file to automatically selected ones. +ac_site_file1=NONE +ac_site_file2=NONE +if test -n "$CONFIG_SITE"; then + ac_site_file1=$CONFIG_SITE +elif test "x$prefix" != xNONE; then + ac_site_file1=$prefix/share/config.site + ac_site_file2=$prefix/etc/config.site +else + ac_site_file1=$ac_default_prefix/share/config.site + ac_site_file2=$ac_default_prefix/etc/config.site +fi +for ac_site_file in "$ac_site_file1" "$ac_site_file2" +do + test "x$ac_site_file" = xNONE && continue + if test -r "$ac_site_file"; then + { $as_echo "$as_me:$LINENO: loading site script $ac_site_file" >&5 +$as_echo "$as_me: loading site script $ac_site_file" >&6;} + sed 's/^/| /' "$ac_site_file" >&5 + . "$ac_site_file" + fi +done + +if test -r "$cache_file"; then + # Some versions of bash will fail to source /dev/null (special + # files actually), so we avoid doing that. + if test -f "$cache_file"; then + { $as_echo "$as_me:$LINENO: loading cache $cache_file" >&5 +$as_echo "$as_me: loading cache $cache_file" >&6;} + case $cache_file in + [\\/]* | ?:[\\/]* ) . "$cache_file";; + *) . "./$cache_file";; + esac + fi +else + { $as_echo "$as_me:$LINENO: creating cache $cache_file" >&5 +$as_echo "$as_me: creating cache $cache_file" >&6;} + >$cache_file +fi + +# Check that the precious variables saved in the cache have kept the same +# value. +ac_cache_corrupted=false +for ac_var in $ac_precious_vars; do + eval ac_old_set=\$ac_cv_env_${ac_var}_set + eval ac_new_set=\$ac_env_${ac_var}_set + eval ac_old_val=\$ac_cv_env_${ac_var}_value + eval ac_new_val=\$ac_env_${ac_var}_value + case $ac_old_set,$ac_new_set in + set,) + { $as_echo "$as_me:$LINENO: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&5 +$as_echo "$as_me: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&2;} + ac_cache_corrupted=: ;; + ,set) + { $as_echo "$as_me:$LINENO: error: \`$ac_var' was not set in the previous run" >&5 +$as_echo "$as_me: error: \`$ac_var' was not set in the previous run" >&2;} + ac_cache_corrupted=: ;; + ,);; + *) + if test "x$ac_old_val" != "x$ac_new_val"; then + # differences in whitespace do not lead to failure. + ac_old_val_w=`echo x $ac_old_val` + ac_new_val_w=`echo x $ac_new_val` + if test "$ac_old_val_w" != "$ac_new_val_w"; then + { $as_echo "$as_me:$LINENO: error: \`$ac_var' has changed since the previous run:" >&5 +$as_echo "$as_me: error: \`$ac_var' has changed since the previous run:" >&2;} + ac_cache_corrupted=: + else + { $as_echo "$as_me:$LINENO: warning: ignoring whitespace changes in \`$ac_var' since the previous run:" >&5 +$as_echo "$as_me: warning: ignoring whitespace changes in \`$ac_var' since the previous run:" >&2;} + eval $ac_var=\$ac_old_val + fi + { $as_echo "$as_me:$LINENO: former value: \`$ac_old_val'" >&5 +$as_echo "$as_me: former value: \`$ac_old_val'" >&2;} + { $as_echo "$as_me:$LINENO: current value: \`$ac_new_val'" >&5 +$as_echo "$as_me: current value: \`$ac_new_val'" >&2;} + fi;; + esac + # Pass precious variables to config.status. + if test "$ac_new_set" = set; then + case $ac_new_val in + *\'*) ac_arg=$ac_var=`$as_echo "$ac_new_val" | sed "s/'/'\\\\\\\\''/g"` ;; + *) ac_arg=$ac_var=$ac_new_val ;; + esac + case " $ac_configure_args " in + *" '$ac_arg' "*) ;; # Avoid dups. Use of quotes ensures accuracy. + *) ac_configure_args="$ac_configure_args '$ac_arg'" ;; + esac + fi +done +if $ac_cache_corrupted; then + { $as_echo "$as_me:$LINENO: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} + { $as_echo "$as_me:$LINENO: error: changes in the environment can compromise the build" >&5 +$as_echo "$as_me: error: changes in the environment can compromise the build" >&2;} + { { $as_echo "$as_me:$LINENO: error: run \`make distclean' and/or \`rm $cache_file' and start over" >&5 +$as_echo "$as_me: error: run \`make distclean' and/or \`rm $cache_file' and start over" >&2;} + { (exit 1); exit 1; }; } +fi + + + + + + + + + + + + + + + + + + + + + + + + + +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + +am__api_version="1.9" +ac_aux_dir= +for ac_dir in "$srcdir" "$srcdir/.." "$srcdir/../.."; do + if test -f "$ac_dir/install-sh"; then + ac_aux_dir=$ac_dir + ac_install_sh="$ac_aux_dir/install-sh -c" + break + elif test -f "$ac_dir/install.sh"; then + ac_aux_dir=$ac_dir + ac_install_sh="$ac_aux_dir/install.sh -c" + break + elif test -f "$ac_dir/shtool"; then + ac_aux_dir=$ac_dir + ac_install_sh="$ac_aux_dir/shtool install -c" + break + fi +done +if test -z "$ac_aux_dir"; then + { { $as_echo "$as_me:$LINENO: error: cannot find install-sh or install.sh in \"$srcdir\" \"$srcdir/..\" \"$srcdir/../..\"" >&5 +$as_echo "$as_me: error: cannot find install-sh or install.sh in \"$srcdir\" \"$srcdir/..\" \"$srcdir/../..\"" >&2;} + { (exit 1); exit 1; }; } +fi + +# These three variables are undocumented and unsupported, +# and are intended to be withdrawn in a future Autoconf release. +# They can cause serious problems if a builder's source tree is in a directory +# whose full name contains unusual characters. +ac_config_guess="$SHELL $ac_aux_dir/config.guess" # Please don't use this var. +ac_config_sub="$SHELL $ac_aux_dir/config.sub" # Please don't use this var. +ac_configure="$SHELL $ac_aux_dir/configure" # Please don't use this var. + + +# Find a good install program. We prefer a C program (faster), +# so one script is as good as another. But avoid the broken or +# incompatible versions: +# SysV /etc/install, /usr/sbin/install +# SunOS /usr/etc/install +# IRIX /sbin/install +# AIX /bin/install +# AmigaOS /C/install, which installs bootblocks on floppy discs +# AIX 4 /usr/bin/installbsd, which doesn't work without a -g flag +# AFS /usr/afsws/bin/install, which mishandles nonexistent args +# SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff" +# OS/2's system install, which has a completely different semantic +# ./install, which can be erroneously created by make from ./install.sh. +# Reject install programs that cannot install multiple files. +{ $as_echo "$as_me:$LINENO: checking for a BSD-compatible install" >&5 +$as_echo_n "checking for a BSD-compatible install... " >&6; } +if test -z "$INSTALL"; then +if test "${ac_cv_path_install+set}" = set; then + $as_echo_n "(cached) " >&6 +else + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + # Account for people who put trailing slashes in PATH elements. +case $as_dir/ in + ./ | .// | /cC/* | \ + /etc/* | /usr/sbin/* | /usr/etc/* | /sbin/* | /usr/afsws/bin/* | \ + ?:\\/os2\\/install\\/* | ?:\\/OS2\\/INSTALL\\/* | \ + /usr/ucb/* ) ;; + *) + # OSF1 and SCO ODT 3.0 have their own names for install. + # Don't use installbsd from OSF since it installs stuff as root + # by default. + for ac_prog in ginstall scoinst install; do + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_prog$ac_exec_ext" && $as_test_x "$as_dir/$ac_prog$ac_exec_ext"; }; then + if test $ac_prog = install && + grep dspmsg "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then + # AIX install. It has an incompatible calling convention. + : + elif test $ac_prog = install && + grep pwplus "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then + # program-specific install script used by HP pwplus--don't use. + : + else + rm -rf conftest.one conftest.two conftest.dir + echo one > conftest.one + echo two > conftest.two + mkdir conftest.dir + if "$as_dir/$ac_prog$ac_exec_ext" -c conftest.one conftest.two "`pwd`/conftest.dir" && + test -s conftest.one && test -s conftest.two && + test -s conftest.dir/conftest.one && + test -s conftest.dir/conftest.two + then + ac_cv_path_install="$as_dir/$ac_prog$ac_exec_ext -c" + break 3 + fi + fi + fi + done + done + ;; +esac + +done +IFS=$as_save_IFS + +rm -rf conftest.one conftest.two conftest.dir + +fi + if test "${ac_cv_path_install+set}" = set; then + INSTALL=$ac_cv_path_install + else + # As a last resort, use the slow shell script. Don't cache a + # value for INSTALL within a source directory, because that will + # break other packages using the cache if that directory is + # removed, or if the value is a relative name. + INSTALL=$ac_install_sh + fi +fi +{ $as_echo "$as_me:$LINENO: result: $INSTALL" >&5 +$as_echo "$INSTALL" >&6; } + +# Use test -z because SunOS4 sh mishandles braces in ${var-val}. +# It thinks the first close brace ends the variable substitution. +test -z "$INSTALL_PROGRAM" && INSTALL_PROGRAM='${INSTALL}' + +test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL}' + +test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644' + +{ $as_echo "$as_me:$LINENO: checking whether build environment is sane" >&5 +$as_echo_n "checking whether build environment is sane... " >&6; } +# Just in case +sleep 1 +echo timestamp > conftest.file +# Do `set' in a subshell so we don't clobber the current shell's +# arguments. Must try -L first in case configure is actually a +# symlink; some systems play weird games with the mod time of symlinks +# (eg FreeBSD returns the mod time of the symlink's containing +# directory). +if ( + set X `ls -Lt $srcdir/configure conftest.file 2> /dev/null` + if test "$*" = "X"; then + # -L didn't work. + set X `ls -t $srcdir/configure conftest.file` + fi + rm -f conftest.file + if test "$*" != "X $srcdir/configure conftest.file" \ + && test "$*" != "X conftest.file $srcdir/configure"; then + + # If neither matched, then we have a broken ls. This can happen + # if, for instance, CONFIG_SHELL is bash and it inherits a + # broken ls alias from the environment. This has actually + # happened. Such a system could not be considered "sane". + { { $as_echo "$as_me:$LINENO: error: ls -t appears to fail. Make sure there is not a broken +alias in your environment" >&5 +$as_echo "$as_me: error: ls -t appears to fail. Make sure there is not a broken +alias in your environment" >&2;} + { (exit 1); exit 1; }; } + fi + + test "$2" = conftest.file + ) +then + # Ok. + : +else + { { $as_echo "$as_me:$LINENO: error: newly created file is older than distributed files! +Check your system clock" >&5 +$as_echo "$as_me: error: newly created file is older than distributed files! +Check your system clock" >&2;} + { (exit 1); exit 1; }; } +fi +{ $as_echo "$as_me:$LINENO: result: yes" >&5 +$as_echo "yes" >&6; } +test "$program_prefix" != NONE && + program_transform_name="s&^&$program_prefix&;$program_transform_name" +# Use a double $ so make ignores it. +test "$program_suffix" != NONE && + program_transform_name="s&\$&$program_suffix&;$program_transform_name" +# Double any \ or $. +# By default was `s,x,x', remove it if useless. +ac_script='s/[\\$]/&&/g;s/;s,x,x,$//' +program_transform_name=`$as_echo "$program_transform_name" | sed "$ac_script"` + +# expand $ac_aux_dir to an absolute path +am_aux_dir=`cd $ac_aux_dir && pwd` + +test x"${MISSING+set}" = xset || MISSING="\${SHELL} $am_aux_dir/missing" +# Use eval to expand $SHELL +if eval "$MISSING --run true"; then + am_missing_run="$MISSING --run " +else + am_missing_run= + { $as_echo "$as_me:$LINENO: WARNING: \`missing' script is too old or missing" >&5 +$as_echo "$as_me: WARNING: \`missing' script is too old or missing" >&2;} +fi + +if mkdir -p --version . >/dev/null 2>&1 && test ! -d ./--version; then + # We used to keeping the `.' as first argument, in order to + # allow $(mkdir_p) to be used without argument. As in + # $(mkdir_p) $(somedir) + # where $(somedir) is conditionally defined. However this is wrong + # for two reasons: + # 1. if the package is installed by a user who cannot write `.' + # make install will fail, + # 2. the above comment should most certainly read + # $(mkdir_p) $(DESTDIR)$(somedir) + # so it does not work when $(somedir) is undefined and + # $(DESTDIR) is not. + # To support the latter case, we have to write + # test -z "$(somedir)" || $(mkdir_p) $(DESTDIR)$(somedir), + # so the `.' trick is pointless. + mkdir_p='mkdir -p --' +else + # On NextStep and OpenStep, the `mkdir' command does not + # recognize any option. It will interpret all options as + # directories to create, and then abort because `.' already + # exists. + for d in ./-p ./--version; + do + test -d $d && rmdir $d + done + # $(mkinstalldirs) is defined by Automake if mkinstalldirs exists. + if test -f "$ac_aux_dir/mkinstalldirs"; then + mkdir_p='$(mkinstalldirs)' + else + mkdir_p='$(install_sh) -d' + fi +fi + +for ac_prog in gawk mawk nawk awk +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:$LINENO: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if test "${ac_cv_prog_AWK+set}" = set; then + $as_echo_n "(cached) " >&6 +else + if test -n "$AWK"; then + ac_cv_prog_AWK="$AWK" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_AWK="$ac_prog" + $as_echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done +IFS=$as_save_IFS + +fi +fi +AWK=$ac_cv_prog_AWK +if test -n "$AWK"; then + { $as_echo "$as_me:$LINENO: result: $AWK" >&5 +$as_echo "$AWK" >&6; } +else + { $as_echo "$as_me:$LINENO: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$AWK" && break +done + +{ $as_echo "$as_me:$LINENO: checking whether ${MAKE-make} sets \$(MAKE)" >&5 +$as_echo_n "checking whether ${MAKE-make} sets \$(MAKE)... " >&6; } +set x ${MAKE-make} +ac_make=`$as_echo "$2" | sed 's/+/p/g; s/[^a-zA-Z0-9_]/_/g'` +if { as_var=ac_cv_prog_make_${ac_make}_set; eval "test \"\${$as_var+set}\" = set"; }; then + $as_echo_n "(cached) " >&6 +else + cat >conftest.make <<\_ACEOF +SHELL = /bin/sh +all: + @echo '@@@%%%=$(MAKE)=@@@%%%' +_ACEOF +# GNU make sometimes prints "make[1]: Entering...", which would confuse us. +case `${MAKE-make} -f conftest.make 2>/dev/null` in + *@@@%%%=?*=@@@%%%*) + eval ac_cv_prog_make_${ac_make}_set=yes;; + *) + eval ac_cv_prog_make_${ac_make}_set=no;; +esac +rm -f conftest.make +fi +if eval test \$ac_cv_prog_make_${ac_make}_set = yes; then + { $as_echo "$as_me:$LINENO: result: yes" >&5 +$as_echo "yes" >&6; } + SET_MAKE= +else + { $as_echo "$as_me:$LINENO: result: no" >&5 +$as_echo "no" >&6; } + SET_MAKE="MAKE=${MAKE-make}" +fi + +rm -rf .tst 2>/dev/null +mkdir .tst 2>/dev/null +if test -d .tst; then + am__leading_dot=. +else + am__leading_dot=_ +fi +rmdir .tst 2>/dev/null + +# test to see if srcdir already configured +if test "`cd $srcdir && pwd`" != "`pwd`" && + test -f $srcdir/config.status; then + { { $as_echo "$as_me:$LINENO: error: source directory already configured; run \"make distclean\" there first" >&5 +$as_echo "$as_me: error: source directory already configured; run \"make distclean\" there first" >&2;} + { (exit 1); exit 1; }; } +fi + +# test whether we have cygpath +if test -z "$CYGPATH_W"; then + if (cygpath --version) >/dev/null 2>/dev/null; then + CYGPATH_W='cygpath -w' + else + CYGPATH_W=echo + fi +fi + + +# Define the identity of the package. + PACKAGE='memscore' + VERSION='1.0' + + +cat >>confdefs.h <<_ACEOF +#define PACKAGE "$PACKAGE" +_ACEOF + + +cat >>confdefs.h <<_ACEOF +#define VERSION "$VERSION" +_ACEOF + +# Some tools Automake needs. + +ACLOCAL=${ACLOCAL-"${am_missing_run}aclocal-${am__api_version}"} + + +AUTOCONF=${AUTOCONF-"${am_missing_run}autoconf"} + + +AUTOMAKE=${AUTOMAKE-"${am_missing_run}automake-${am__api_version}"} + + +AUTOHEADER=${AUTOHEADER-"${am_missing_run}autoheader"} + + +MAKEINFO=${MAKEINFO-"${am_missing_run}makeinfo"} + +install_sh=${install_sh-"$am_aux_dir/install-sh"} + +# Installed binaries are usually stripped using `strip' when the user +# run `make install-strip'. However `strip' might not be the right +# tool to use in cross-compilation environments, therefore Automake +# will honor the `STRIP' environment variable to overrule this program. +if test "$cross_compiling" != no; then + if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}strip", so it can be a program name with args. +set dummy ${ac_tool_prefix}strip; ac_word=$2 +{ $as_echo "$as_me:$LINENO: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if test "${ac_cv_prog_STRIP+set}" = set; then + $as_echo_n "(cached) " >&6 +else + if test -n "$STRIP"; then + ac_cv_prog_STRIP="$STRIP" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_STRIP="${ac_tool_prefix}strip" + $as_echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done +IFS=$as_save_IFS + +fi +fi +STRIP=$ac_cv_prog_STRIP +if test -n "$STRIP"; then + { $as_echo "$as_me:$LINENO: result: $STRIP" >&5 +$as_echo "$STRIP" >&6; } +else + { $as_echo "$as_me:$LINENO: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_prog_STRIP"; then + ac_ct_STRIP=$STRIP + # Extract the first word of "strip", so it can be a program name with args. +set dummy strip; ac_word=$2 +{ $as_echo "$as_me:$LINENO: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if test "${ac_cv_prog_ac_ct_STRIP+set}" = set; then + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_STRIP"; then + ac_cv_prog_ac_ct_STRIP="$ac_ct_STRIP" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_STRIP="strip" + $as_echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done +IFS=$as_save_IFS + +fi +fi +ac_ct_STRIP=$ac_cv_prog_ac_ct_STRIP +if test -n "$ac_ct_STRIP"; then + { $as_echo "$as_me:$LINENO: result: $ac_ct_STRIP" >&5 +$as_echo "$ac_ct_STRIP" >&6; } +else + { $as_echo "$as_me:$LINENO: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_ct_STRIP" = x; then + STRIP=":" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:$LINENO: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + STRIP=$ac_ct_STRIP + fi +else + STRIP="$ac_cv_prog_STRIP" +fi + +fi +INSTALL_STRIP_PROGRAM="\${SHELL} \$(install_sh) -c -s" + +# We need awk for the "check" target. The system "awk" is bad on +# some platforms. +# Always define AMTAR for backward compatibility. + +AMTAR=${AMTAR-"${am_missing_run}tar"} + +am__tar='${AMTAR} chof - "$$tardir"'; am__untar='${AMTAR} xf -' + + + + + +ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu + + + +# Check whether --with-irstlm was given. +if test "${with_irstlm+set}" = set; then + withval=$with_irstlm; with_irstlm=$withval +else + with_irstlm=check +fi + + + +# Check whether --with-gsl was given. +if test "${with_gsl+set}" = set; then + withval=$with_gsl; with_gsl=$withval + CPPFLAGS="$CPPFLAGS -I$with_gsl/include" + LDFLAGS="$LDFLAGS -L$with_gsl/lib" +else + with_gsl=check +fi + + +# Check whether --enable-channel was given. +if test "${enable_channel+set}" = set; then + enableval=$enable_channel; +cat >>confdefs.h <<\_ACEOF +#define ENABLE_CHANNEL_SCORER /**/ +_ACEOF + +else + enable_channel=no +fi + + + + +ac_config_headers="$ac_config_headers config.h" + + +# Checks for programs. +ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu +if test -z "$CXX"; then + if test -n "$CCC"; then + CXX=$CCC + else + if test -n "$ac_tool_prefix"; then + for ac_prog in g++ c++ gpp aCC CC cxx cc++ cl.exe FCC KCC RCC xlC_r xlC + do + # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. +set dummy $ac_tool_prefix$ac_prog; ac_word=$2 +{ $as_echo "$as_me:$LINENO: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if test "${ac_cv_prog_CXX+set}" = set; then + $as_echo_n "(cached) " >&6 +else + if test -n "$CXX"; then + ac_cv_prog_CXX="$CXX" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_CXX="$ac_tool_prefix$ac_prog" + $as_echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done +IFS=$as_save_IFS + +fi +fi +CXX=$ac_cv_prog_CXX +if test -n "$CXX"; then + { $as_echo "$as_me:$LINENO: result: $CXX" >&5 +$as_echo "$CXX" >&6; } +else + { $as_echo "$as_me:$LINENO: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$CXX" && break + done +fi +if test -z "$CXX"; then + ac_ct_CXX=$CXX + for ac_prog in g++ c++ gpp aCC CC cxx cc++ cl.exe FCC KCC RCC xlC_r xlC +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:$LINENO: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if test "${ac_cv_prog_ac_ct_CXX+set}" = set; then + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_CXX"; then + ac_cv_prog_ac_ct_CXX="$ac_ct_CXX" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_CXX="$ac_prog" + $as_echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done +IFS=$as_save_IFS + +fi +fi +ac_ct_CXX=$ac_cv_prog_ac_ct_CXX +if test -n "$ac_ct_CXX"; then + { $as_echo "$as_me:$LINENO: result: $ac_ct_CXX" >&5 +$as_echo "$ac_ct_CXX" >&6; } +else + { $as_echo "$as_me:$LINENO: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$ac_ct_CXX" && break +done + + if test "x$ac_ct_CXX" = x; then + CXX="g++" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:$LINENO: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + CXX=$ac_ct_CXX + fi +fi + + fi +fi +# Provide some information about the compiler. +$as_echo "$as_me:$LINENO: checking for C++ compiler version" >&5 +set X $ac_compile +ac_compiler=$2 +{ (ac_try="$ac_compiler --version >&5" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compiler --version >&5") 2>&5 + ac_status=$? + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } +{ (ac_try="$ac_compiler -v >&5" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compiler -v >&5") 2>&5 + ac_status=$? + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } +{ (ac_try="$ac_compiler -V >&5" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compiler -V >&5") 2>&5 + ac_status=$? + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } + +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +ac_clean_files_save=$ac_clean_files +ac_clean_files="$ac_clean_files a.out a.out.dSYM a.exe b.out" +# Try to create an executable without -o first, disregard a.out. +# It will help us diagnose broken compilers, and finding out an intuition +# of exeext. +{ $as_echo "$as_me:$LINENO: checking for C++ compiler default output file name" >&5 +$as_echo_n "checking for C++ compiler default output file name... " >&6; } +ac_link_default=`$as_echo "$ac_link" | sed 's/ -o *conftest[^ ]*//'` + +# The possible output files: +ac_files="a.out conftest.exe conftest a.exe a_out.exe b.out conftest.*" + +ac_rmfiles= +for ac_file in $ac_files +do + case $ac_file in + *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) ;; + * ) ac_rmfiles="$ac_rmfiles $ac_file";; + esac +done +rm -f $ac_rmfiles + +if { (ac_try="$ac_link_default" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_link_default") 2>&5 + ac_status=$? + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; then + # Autoconf-2.13 could set the ac_cv_exeext variable to `no'. +# So ignore a value of `no', otherwise this would lead to `EXEEXT = no' +# in a Makefile. We should not override ac_cv_exeext if it was cached, +# so that the user can short-circuit this test for compilers unknown to +# Autoconf. +for ac_file in $ac_files '' +do + test -f "$ac_file" || continue + case $ac_file in + *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) + ;; + [ab].out ) + # We found the default executable, but exeext='' is most + # certainly right. + break;; + *.* ) + if test "${ac_cv_exeext+set}" = set && test "$ac_cv_exeext" != no; + then :; else + ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'` + fi + # We set ac_cv_exeext here because the later test for it is not + # safe: cross compilers may not add the suffix if given an `-o' + # argument, so we may need to know it at that point already. + # Even if this section looks crufty: it has the advantage of + # actually working. + break;; + * ) + break;; + esac +done +test "$ac_cv_exeext" = no && ac_cv_exeext= + +else + ac_file='' +fi + +{ $as_echo "$as_me:$LINENO: result: $ac_file" >&5 +$as_echo "$ac_file" >&6; } +if test -z "$ac_file"; then + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +{ { $as_echo "$as_me:$LINENO: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +{ { $as_echo "$as_me:$LINENO: error: C++ compiler cannot create executables +See \`config.log' for more details." >&5 +$as_echo "$as_me: error: C++ compiler cannot create executables +See \`config.log' for more details." >&2;} + { (exit 77); exit 77; }; }; } +fi + +ac_exeext=$ac_cv_exeext + +# Check that the compiler produces executables we can run. If not, either +# the compiler is broken, or we cross compile. +{ $as_echo "$as_me:$LINENO: checking whether the C++ compiler works" >&5 +$as_echo_n "checking whether the C++ compiler works... " >&6; } +# FIXME: These cross compiler hacks should be removed for Autoconf 3.0 +# If not cross compiling, check that we can run a simple program. +if test "$cross_compiling" != yes; then + if { ac_try='./$ac_file' + { (case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_try") 2>&5 + ac_status=$? + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + cross_compiling=no + else + if test "$cross_compiling" = maybe; then + cross_compiling=yes + else + { { $as_echo "$as_me:$LINENO: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +{ { $as_echo "$as_me:$LINENO: error: cannot run C++ compiled programs. +If you meant to cross compile, use \`--host'. +See \`config.log' for more details." >&5 +$as_echo "$as_me: error: cannot run C++ compiled programs. +If you meant to cross compile, use \`--host'. +See \`config.log' for more details." >&2;} + { (exit 1); exit 1; }; }; } + fi + fi +fi +{ $as_echo "$as_me:$LINENO: result: yes" >&5 +$as_echo "yes" >&6; } + +rm -f -r a.out a.out.dSYM a.exe conftest$ac_cv_exeext b.out +ac_clean_files=$ac_clean_files_save +# Check that the compiler produces executables we can run. If not, either +# the compiler is broken, or we cross compile. +{ $as_echo "$as_me:$LINENO: checking whether we are cross compiling" >&5 +$as_echo_n "checking whether we are cross compiling... " >&6; } +{ $as_echo "$as_me:$LINENO: result: $cross_compiling" >&5 +$as_echo "$cross_compiling" >&6; } + +{ $as_echo "$as_me:$LINENO: checking for suffix of executables" >&5 +$as_echo_n "checking for suffix of executables... " >&6; } +if { (ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_link") 2>&5 + ac_status=$? + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; then + # If both `conftest.exe' and `conftest' are `present' (well, observable) +# catch `conftest.exe'. For instance with Cygwin, `ls conftest' will +# work properly (i.e., refer to `conftest.exe'), while it won't with +# `rm'. +for ac_file in conftest.exe conftest conftest.*; do + test -f "$ac_file" || continue + case $ac_file in + *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) ;; + *.* ) ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'` + break;; + * ) break;; + esac +done +else + { { $as_echo "$as_me:$LINENO: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +{ { $as_echo "$as_me:$LINENO: error: cannot compute suffix of executables: cannot compile and link +See \`config.log' for more details." >&5 +$as_echo "$as_me: error: cannot compute suffix of executables: cannot compile and link +See \`config.log' for more details." >&2;} + { (exit 1); exit 1; }; }; } +fi + +rm -f conftest$ac_cv_exeext +{ $as_echo "$as_me:$LINENO: result: $ac_cv_exeext" >&5 +$as_echo "$ac_cv_exeext" >&6; } + +rm -f conftest.$ac_ext +EXEEXT=$ac_cv_exeext +ac_exeext=$EXEEXT +{ $as_echo "$as_me:$LINENO: checking for suffix of object files" >&5 +$as_echo_n "checking for suffix of object files... " >&6; } +if test "${ac_cv_objext+set}" = set; then + $as_echo_n "(cached) " >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +rm -f conftest.o conftest.obj +if { (ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compile") 2>&5 + ac_status=$? + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; then + for ac_file in conftest.o conftest.obj conftest.*; do + test -f "$ac_file" || continue; + case $ac_file in + *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM ) ;; + *) ac_cv_objext=`expr "$ac_file" : '.*\.\(.*\)'` + break;; + esac +done +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +{ { $as_echo "$as_me:$LINENO: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +{ { $as_echo "$as_me:$LINENO: error: cannot compute suffix of object files: cannot compile +See \`config.log' for more details." >&5 +$as_echo "$as_me: error: cannot compute suffix of object files: cannot compile +See \`config.log' for more details." >&2;} + { (exit 1); exit 1; }; }; } +fi + +rm -f conftest.$ac_cv_objext conftest.$ac_ext +fi +{ $as_echo "$as_me:$LINENO: result: $ac_cv_objext" >&5 +$as_echo "$ac_cv_objext" >&6; } +OBJEXT=$ac_cv_objext +ac_objext=$OBJEXT +{ $as_echo "$as_me:$LINENO: checking whether we are using the GNU C++ compiler" >&5 +$as_echo_n "checking whether we are using the GNU C++ compiler... " >&6; } +if test "${ac_cv_cxx_compiler_gnu+set}" = set; then + $as_echo_n "(cached) " >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +int +main () +{ +#ifndef __GNUC__ + choke me +#endif + + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + ac_compiler_gnu=yes +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_compiler_gnu=no +fi + +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +ac_cv_cxx_compiler_gnu=$ac_compiler_gnu + +fi +{ $as_echo "$as_me:$LINENO: result: $ac_cv_cxx_compiler_gnu" >&5 +$as_echo "$ac_cv_cxx_compiler_gnu" >&6; } +if test $ac_compiler_gnu = yes; then + GXX=yes +else + GXX= +fi +ac_test_CXXFLAGS=${CXXFLAGS+set} +ac_save_CXXFLAGS=$CXXFLAGS +{ $as_echo "$as_me:$LINENO: checking whether $CXX accepts -g" >&5 +$as_echo_n "checking whether $CXX accepts -g... " >&6; } +if test "${ac_cv_prog_cxx_g+set}" = set; then + $as_echo_n "(cached) " >&6 +else + ac_save_cxx_werror_flag=$ac_cxx_werror_flag + ac_cxx_werror_flag=yes + ac_cv_prog_cxx_g=no + CXXFLAGS="-g" + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + ac_cv_prog_cxx_g=yes +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + CXXFLAGS="" + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + : +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_cxx_werror_flag=$ac_save_cxx_werror_flag + CXXFLAGS="-g" + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + ac_cv_prog_cxx_g=yes +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + +fi + +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi + +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi + +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + ac_cxx_werror_flag=$ac_save_cxx_werror_flag +fi +{ $as_echo "$as_me:$LINENO: result: $ac_cv_prog_cxx_g" >&5 +$as_echo "$ac_cv_prog_cxx_g" >&6; } +if test "$ac_test_CXXFLAGS" = set; then + CXXFLAGS=$ac_save_CXXFLAGS +elif test $ac_cv_prog_cxx_g = yes; then + if test "$GXX" = yes; then + CXXFLAGS="-g -O2" + else + CXXFLAGS="-g" + fi +else + if test "$GXX" = yes; then + CXXFLAGS="-O2" + else + CXXFLAGS= + fi +fi +ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu +DEPDIR="${am__leading_dot}deps" + +ac_config_commands="$ac_config_commands depfiles" + + +am_make=${MAKE-make} +cat > confinc << 'END' +am__doit: + @echo done +.PHONY: am__doit +END +# If we don't find an include directive, just comment out the code. +{ $as_echo "$as_me:$LINENO: checking for style of include used by $am_make" >&5 +$as_echo_n "checking for style of include used by $am_make... " >&6; } +am__include="#" +am__quote= +_am_result=none +# First try GNU make style include. +echo "include confinc" > confmf +# We grep out `Entering directory' and `Leaving directory' +# messages which can occur if `w' ends up in MAKEFLAGS. +# In particular we don't look at `^make:' because GNU make might +# be invoked under some other name (usually "gmake"), in which +# case it prints its new name instead of `make'. +if test "`$am_make -s -f confmf 2> /dev/null | grep -v 'ing directory'`" = "done"; then + am__include=include + am__quote= + _am_result=GNU +fi +# Now try BSD make style include. +if test "$am__include" = "#"; then + echo '.include "confinc"' > confmf + if test "`$am_make -s -f confmf 2> /dev/null`" = "done"; then + am__include=.include + am__quote="\"" + _am_result=BSD + fi +fi + + +{ $as_echo "$as_me:$LINENO: result: $_am_result" >&5 +$as_echo "$_am_result" >&6; } +rm -f confinc confmf + +# Check whether --enable-dependency-tracking was given. +if test "${enable_dependency_tracking+set}" = set; then + enableval=$enable_dependency_tracking; +fi + +if test "x$enable_dependency_tracking" != xno; then + am_depcomp="$ac_aux_dir/depcomp" + AMDEPBACKSLASH='\' +fi + + +if test "x$enable_dependency_tracking" != xno; then + AMDEP_TRUE= + AMDEP_FALSE='#' +else + AMDEP_TRUE='#' + AMDEP_FALSE= +fi + + + + +depcc="$CXX" am_compiler_list= + +{ $as_echo "$as_me:$LINENO: checking dependency style of $depcc" >&5 +$as_echo_n "checking dependency style of $depcc... " >&6; } +if test "${am_cv_CXX_dependencies_compiler_type+set}" = set; then + $as_echo_n "(cached) " >&6 +else + if test -z "$AMDEP_TRUE" && test -f "$am_depcomp"; then + # We make a subdir and do the tests there. Otherwise we can end up + # making bogus files that we don't know about and never remove. For + # instance it was reported that on HP-UX the gcc test will end up + # making a dummy file named `D' -- because `-MD' means `put the output + # in D'. + mkdir conftest.dir + # Copy depcomp to subdir because otherwise we won't find it if we're + # using a relative directory. + cp "$am_depcomp" conftest.dir + cd conftest.dir + # We will build objects and dependencies in a subdirectory because + # it helps to detect inapplicable dependency modes. For instance + # both Tru64's cc and ICC support -MD to output dependencies as a + # side effect of compilation, but ICC will put the dependencies in + # the current directory while Tru64 will put them in the object + # directory. + mkdir sub + + am_cv_CXX_dependencies_compiler_type=none + if test "$am_compiler_list" = ""; then + am_compiler_list=`sed -n 's/^#*\([a-zA-Z0-9]*\))$/\1/p' < ./depcomp` + fi + for depmode in $am_compiler_list; do + # Setup a source with many dependencies, because some compilers + # like to wrap large dependency lists on column 80 (with \), and + # we should not choose a depcomp mode which is confused by this. + # + # We need to recreate these files for each test, as the compiler may + # overwrite some of them when testing with obscure command lines. + # This happens at least with the AIX C compiler. + : > sub/conftest.c + for i in 1 2 3 4 5 6; do + echo '#include "conftst'$i'.h"' >> sub/conftest.c + # Using `: > sub/conftst$i.h' creates only sub/conftst1.h with + # Solaris 8's {/usr,}/bin/sh. + touch sub/conftst$i.h + done + echo "${am__include} ${am__quote}sub/conftest.Po${am__quote}" > confmf + + case $depmode in + nosideeffect) + # after this tag, mechanisms are not by side-effect, so they'll + # only be used when explicitly requested + if test "x$enable_dependency_tracking" = xyes; then + continue + else + break + fi + ;; + none) break ;; + esac + # We check with `-c' and `-o' for the sake of the "dashmstdout" + # mode. It turns out that the SunPro C++ compiler does not properly + # handle `-M -o', and we need to detect this. + if depmode=$depmode \ + source=sub/conftest.c object=sub/conftest.${OBJEXT-o} \ + depfile=sub/conftest.Po tmpdepfile=sub/conftest.TPo \ + $SHELL ./depcomp $depcc -c -o sub/conftest.${OBJEXT-o} sub/conftest.c \ + >/dev/null 2>conftest.err && + grep sub/conftst6.h sub/conftest.Po > /dev/null 2>&1 && + grep sub/conftest.${OBJEXT-o} sub/conftest.Po > /dev/null 2>&1 && + ${MAKE-make} -s -f confmf > /dev/null 2>&1; then + # icc doesn't choke on unknown options, it will just issue warnings + # or remarks (even with -Werror). So we grep stderr for any message + # that says an option was ignored or not supported. + # When given -MP, icc 7.0 and 7.1 complain thusly: + # icc: Command line warning: ignoring option '-M'; no argument required + # The diagnosis changed in icc 8.0: + # icc: Command line remark: option '-MP' not supported + if (grep 'ignoring option' conftest.err || + grep 'not supported' conftest.err) >/dev/null 2>&1; then :; else + am_cv_CXX_dependencies_compiler_type=$depmode + break + fi + fi + done + + cd .. + rm -rf conftest.dir +else + am_cv_CXX_dependencies_compiler_type=none +fi + +fi +{ $as_echo "$as_me:$LINENO: result: $am_cv_CXX_dependencies_compiler_type" >&5 +$as_echo "$am_cv_CXX_dependencies_compiler_type" >&6; } +CXXDEPMODE=depmode=$am_cv_CXX_dependencies_compiler_type + + + +if + test "x$enable_dependency_tracking" != xno \ + && test "$am_cv_CXX_dependencies_compiler_type" = gcc3; then + am__fastdepCXX_TRUE= + am__fastdepCXX_FALSE='#' +else + am__fastdepCXX_TRUE='#' + am__fastdepCXX_FALSE= +fi + + +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu +if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}gcc", so it can be a program name with args. +set dummy ${ac_tool_prefix}gcc; ac_word=$2 +{ $as_echo "$as_me:$LINENO: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if test "${ac_cv_prog_CC+set}" = set; then + $as_echo_n "(cached) " >&6 +else + if test -n "$CC"; then + ac_cv_prog_CC="$CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_CC="${ac_tool_prefix}gcc" + $as_echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done +IFS=$as_save_IFS + +fi +fi +CC=$ac_cv_prog_CC +if test -n "$CC"; then + { $as_echo "$as_me:$LINENO: result: $CC" >&5 +$as_echo "$CC" >&6; } +else + { $as_echo "$as_me:$LINENO: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_prog_CC"; then + ac_ct_CC=$CC + # Extract the first word of "gcc", so it can be a program name with args. +set dummy gcc; ac_word=$2 +{ $as_echo "$as_me:$LINENO: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if test "${ac_cv_prog_ac_ct_CC+set}" = set; then + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_CC"; then + ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_CC="gcc" + $as_echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done +IFS=$as_save_IFS + +fi +fi +ac_ct_CC=$ac_cv_prog_ac_ct_CC +if test -n "$ac_ct_CC"; then + { $as_echo "$as_me:$LINENO: result: $ac_ct_CC" >&5 +$as_echo "$ac_ct_CC" >&6; } +else + { $as_echo "$as_me:$LINENO: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_ct_CC" = x; then + CC="" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:$LINENO: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + CC=$ac_ct_CC + fi +else + CC="$ac_cv_prog_CC" +fi + +if test -z "$CC"; then + if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}cc", so it can be a program name with args. +set dummy ${ac_tool_prefix}cc; ac_word=$2 +{ $as_echo "$as_me:$LINENO: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if test "${ac_cv_prog_CC+set}" = set; then + $as_echo_n "(cached) " >&6 +else + if test -n "$CC"; then + ac_cv_prog_CC="$CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_CC="${ac_tool_prefix}cc" + $as_echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done +IFS=$as_save_IFS + +fi +fi +CC=$ac_cv_prog_CC +if test -n "$CC"; then + { $as_echo "$as_me:$LINENO: result: $CC" >&5 +$as_echo "$CC" >&6; } +else + { $as_echo "$as_me:$LINENO: result: no" >&5 +$as_echo "no" >&6; } +fi + + + fi +fi +if test -z "$CC"; then + # Extract the first word of "cc", so it can be a program name with args. +set dummy cc; ac_word=$2 +{ $as_echo "$as_me:$LINENO: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if test "${ac_cv_prog_CC+set}" = set; then + $as_echo_n "(cached) " >&6 +else + if test -n "$CC"; then + ac_cv_prog_CC="$CC" # Let the user override the test. +else + ac_prog_rejected=no +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + if test "$as_dir/$ac_word$ac_exec_ext" = "/usr/ucb/cc"; then + ac_prog_rejected=yes + continue + fi + ac_cv_prog_CC="cc" + $as_echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done +IFS=$as_save_IFS + +if test $ac_prog_rejected = yes; then + # We found a bogon in the path, so make sure we never use it. + set dummy $ac_cv_prog_CC + shift + if test $# != 0; then + # We chose a different compiler from the bogus one. + # However, it has the same basename, so the bogon will be chosen + # first if we set CC to just the basename; use the full file name. + shift + ac_cv_prog_CC="$as_dir/$ac_word${1+' '}$@" + fi +fi +fi +fi +CC=$ac_cv_prog_CC +if test -n "$CC"; then + { $as_echo "$as_me:$LINENO: result: $CC" >&5 +$as_echo "$CC" >&6; } +else + { $as_echo "$as_me:$LINENO: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$CC"; then + if test -n "$ac_tool_prefix"; then + for ac_prog in cl.exe + do + # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. +set dummy $ac_tool_prefix$ac_prog; ac_word=$2 +{ $as_echo "$as_me:$LINENO: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if test "${ac_cv_prog_CC+set}" = set; then + $as_echo_n "(cached) " >&6 +else + if test -n "$CC"; then + ac_cv_prog_CC="$CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_CC="$ac_tool_prefix$ac_prog" + $as_echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done +IFS=$as_save_IFS + +fi +fi +CC=$ac_cv_prog_CC +if test -n "$CC"; then + { $as_echo "$as_me:$LINENO: result: $CC" >&5 +$as_echo "$CC" >&6; } +else + { $as_echo "$as_me:$LINENO: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$CC" && break + done +fi +if test -z "$CC"; then + ac_ct_CC=$CC + for ac_prog in cl.exe +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:$LINENO: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if test "${ac_cv_prog_ac_ct_CC+set}" = set; then + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_CC"; then + ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_CC="$ac_prog" + $as_echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done +IFS=$as_save_IFS + +fi +fi +ac_ct_CC=$ac_cv_prog_ac_ct_CC +if test -n "$ac_ct_CC"; then + { $as_echo "$as_me:$LINENO: result: $ac_ct_CC" >&5 +$as_echo "$ac_ct_CC" >&6; } +else + { $as_echo "$as_me:$LINENO: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$ac_ct_CC" && break +done + + if test "x$ac_ct_CC" = x; then + CC="" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:$LINENO: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + CC=$ac_ct_CC + fi +fi + +fi + + +test -z "$CC" && { { $as_echo "$as_me:$LINENO: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +{ { $as_echo "$as_me:$LINENO: error: no acceptable C compiler found in \$PATH +See \`config.log' for more details." >&5 +$as_echo "$as_me: error: no acceptable C compiler found in \$PATH +See \`config.log' for more details." >&2;} + { (exit 1); exit 1; }; }; } + +# Provide some information about the compiler. +$as_echo "$as_me:$LINENO: checking for C compiler version" >&5 +set X $ac_compile +ac_compiler=$2 +{ (ac_try="$ac_compiler --version >&5" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compiler --version >&5") 2>&5 + ac_status=$? + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } +{ (ac_try="$ac_compiler -v >&5" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compiler -v >&5") 2>&5 + ac_status=$? + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } +{ (ac_try="$ac_compiler -V >&5" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compiler -V >&5") 2>&5 + ac_status=$? + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } + +{ $as_echo "$as_me:$LINENO: checking whether we are using the GNU C compiler" >&5 +$as_echo_n "checking whether we are using the GNU C compiler... " >&6; } +if test "${ac_cv_c_compiler_gnu+set}" = set; then + $as_echo_n "(cached) " >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +int +main () +{ +#ifndef __GNUC__ + choke me +#endif + + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_c_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + ac_compiler_gnu=yes +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_compiler_gnu=no +fi + +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +ac_cv_c_compiler_gnu=$ac_compiler_gnu + +fi +{ $as_echo "$as_me:$LINENO: result: $ac_cv_c_compiler_gnu" >&5 +$as_echo "$ac_cv_c_compiler_gnu" >&6; } +if test $ac_compiler_gnu = yes; then + GCC=yes +else + GCC= +fi +ac_test_CFLAGS=${CFLAGS+set} +ac_save_CFLAGS=$CFLAGS +{ $as_echo "$as_me:$LINENO: checking whether $CC accepts -g" >&5 +$as_echo_n "checking whether $CC accepts -g... " >&6; } +if test "${ac_cv_prog_cc_g+set}" = set; then + $as_echo_n "(cached) " >&6 +else + ac_save_c_werror_flag=$ac_c_werror_flag + ac_c_werror_flag=yes + ac_cv_prog_cc_g=no + CFLAGS="-g" + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_c_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + ac_cv_prog_cc_g=yes +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + CFLAGS="" + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_c_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + : +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_c_werror_flag=$ac_save_c_werror_flag + CFLAGS="-g" + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_c_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + ac_cv_prog_cc_g=yes +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + +fi + +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi + +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi + +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + ac_c_werror_flag=$ac_save_c_werror_flag +fi +{ $as_echo "$as_me:$LINENO: result: $ac_cv_prog_cc_g" >&5 +$as_echo "$ac_cv_prog_cc_g" >&6; } +if test "$ac_test_CFLAGS" = set; then + CFLAGS=$ac_save_CFLAGS +elif test $ac_cv_prog_cc_g = yes; then + if test "$GCC" = yes; then + CFLAGS="-g -O2" + else + CFLAGS="-g" + fi +else + if test "$GCC" = yes; then + CFLAGS="-O2" + else + CFLAGS= + fi +fi +{ $as_echo "$as_me:$LINENO: checking for $CC option to accept ISO C89" >&5 +$as_echo_n "checking for $CC option to accept ISO C89... " >&6; } +if test "${ac_cv_prog_cc_c89+set}" = set; then + $as_echo_n "(cached) " >&6 +else + ac_cv_prog_cc_c89=no +ac_save_CC=$CC +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#include +#include +#include +#include +/* Most of the following tests are stolen from RCS 5.7's src/conf.sh. */ +struct buf { int x; }; +FILE * (*rcsopen) (struct buf *, struct stat *, int); +static char *e (p, i) + char **p; + int i; +{ + return p[i]; +} +static char *f (char * (*g) (char **, int), char **p, ...) +{ + char *s; + va_list v; + va_start (v,p); + s = g (p, va_arg (v,int)); + va_end (v); + return s; +} + +/* OSF 4.0 Compaq cc is some sort of almost-ANSI by default. It has + function prototypes and stuff, but not '\xHH' hex character constants. + These don't provoke an error unfortunately, instead are silently treated + as 'x'. The following induces an error, until -std is added to get + proper ANSI mode. Curiously '\x00'!='x' always comes out true, for an + array size at least. It's necessary to write '\x00'==0 to get something + that's true only with -std. */ +int osf4_cc_array ['\x00' == 0 ? 1 : -1]; + +/* IBM C 6 for AIX is almost-ANSI by default, but it replaces macro parameters + inside strings and character constants. */ +#define FOO(x) 'x' +int xlc6_cc_array[FOO(a) == 'x' ? 1 : -1]; + +int test (int i, double x); +struct s1 {int (*f) (int a);}; +struct s2 {int (*f) (double a);}; +int pairnames (int, char **, FILE *(*)(struct buf *, struct stat *, int), int, int); +int argc; +char **argv; +int +main () +{ +return f (e, argv, 0) != argv[0] || f (e, argv, 1) != argv[1]; + ; + return 0; +} +_ACEOF +for ac_arg in '' -qlanglvl=extc89 -qlanglvl=ansi -std \ + -Ae "-Aa -D_HPUX_SOURCE" "-Xc -D__EXTENSIONS__" +do + CC="$ac_save_CC $ac_arg" + rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_c_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + ac_cv_prog_cc_c89=$ac_arg +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + +fi + +rm -f core conftest.err conftest.$ac_objext + test "x$ac_cv_prog_cc_c89" != "xno" && break +done +rm -f conftest.$ac_ext +CC=$ac_save_CC + +fi +# AC_CACHE_VAL +case "x$ac_cv_prog_cc_c89" in + x) + { $as_echo "$as_me:$LINENO: result: none needed" >&5 +$as_echo "none needed" >&6; } ;; + xno) + { $as_echo "$as_me:$LINENO: result: unsupported" >&5 +$as_echo "unsupported" >&6; } ;; + *) + CC="$CC $ac_cv_prog_cc_c89" + { $as_echo "$as_me:$LINENO: result: $ac_cv_prog_cc_c89" >&5 +$as_echo "$ac_cv_prog_cc_c89" >&6; } ;; +esac + + +ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu + +depcc="$CC" am_compiler_list= + +{ $as_echo "$as_me:$LINENO: checking dependency style of $depcc" >&5 +$as_echo_n "checking dependency style of $depcc... " >&6; } +if test "${am_cv_CC_dependencies_compiler_type+set}" = set; then + $as_echo_n "(cached) " >&6 +else + if test -z "$AMDEP_TRUE" && test -f "$am_depcomp"; then + # We make a subdir and do the tests there. Otherwise we can end up + # making bogus files that we don't know about and never remove. For + # instance it was reported that on HP-UX the gcc test will end up + # making a dummy file named `D' -- because `-MD' means `put the output + # in D'. + mkdir conftest.dir + # Copy depcomp to subdir because otherwise we won't find it if we're + # using a relative directory. + cp "$am_depcomp" conftest.dir + cd conftest.dir + # We will build objects and dependencies in a subdirectory because + # it helps to detect inapplicable dependency modes. For instance + # both Tru64's cc and ICC support -MD to output dependencies as a + # side effect of compilation, but ICC will put the dependencies in + # the current directory while Tru64 will put them in the object + # directory. + mkdir sub + + am_cv_CC_dependencies_compiler_type=none + if test "$am_compiler_list" = ""; then + am_compiler_list=`sed -n 's/^#*\([a-zA-Z0-9]*\))$/\1/p' < ./depcomp` + fi + for depmode in $am_compiler_list; do + # Setup a source with many dependencies, because some compilers + # like to wrap large dependency lists on column 80 (with \), and + # we should not choose a depcomp mode which is confused by this. + # + # We need to recreate these files for each test, as the compiler may + # overwrite some of them when testing with obscure command lines. + # This happens at least with the AIX C compiler. + : > sub/conftest.c + for i in 1 2 3 4 5 6; do + echo '#include "conftst'$i'.h"' >> sub/conftest.c + # Using `: > sub/conftst$i.h' creates only sub/conftst1.h with + # Solaris 8's {/usr,}/bin/sh. + touch sub/conftst$i.h + done + echo "${am__include} ${am__quote}sub/conftest.Po${am__quote}" > confmf + + case $depmode in + nosideeffect) + # after this tag, mechanisms are not by side-effect, so they'll + # only be used when explicitly requested + if test "x$enable_dependency_tracking" = xyes; then + continue + else + break + fi + ;; + none) break ;; + esac + # We check with `-c' and `-o' for the sake of the "dashmstdout" + # mode. It turns out that the SunPro C++ compiler does not properly + # handle `-M -o', and we need to detect this. + if depmode=$depmode \ + source=sub/conftest.c object=sub/conftest.${OBJEXT-o} \ + depfile=sub/conftest.Po tmpdepfile=sub/conftest.TPo \ + $SHELL ./depcomp $depcc -c -o sub/conftest.${OBJEXT-o} sub/conftest.c \ + >/dev/null 2>conftest.err && + grep sub/conftst6.h sub/conftest.Po > /dev/null 2>&1 && + grep sub/conftest.${OBJEXT-o} sub/conftest.Po > /dev/null 2>&1 && + ${MAKE-make} -s -f confmf > /dev/null 2>&1; then + # icc doesn't choke on unknown options, it will just issue warnings + # or remarks (even with -Werror). So we grep stderr for any message + # that says an option was ignored or not supported. + # When given -MP, icc 7.0 and 7.1 complain thusly: + # icc: Command line warning: ignoring option '-M'; no argument required + # The diagnosis changed in icc 8.0: + # icc: Command line remark: option '-MP' not supported + if (grep 'ignoring option' conftest.err || + grep 'not supported' conftest.err) >/dev/null 2>&1; then :; else + am_cv_CC_dependencies_compiler_type=$depmode + break + fi + fi + done + + cd .. + rm -rf conftest.dir +else + am_cv_CC_dependencies_compiler_type=none +fi + +fi +{ $as_echo "$as_me:$LINENO: result: $am_cv_CC_dependencies_compiler_type" >&5 +$as_echo "$am_cv_CC_dependencies_compiler_type" >&6; } +CCDEPMODE=depmode=$am_cv_CC_dependencies_compiler_type + + + +if + test "x$enable_dependency_tracking" != xno \ + && test "$am_cv_CC_dependencies_compiler_type" = gcc3; then + am__fastdepCC_TRUE= + am__fastdepCC_FALSE='#' +else + am__fastdepCC_TRUE='#' + am__fastdepCC_FALSE= +fi + + + +# Checks for libraries. + + + +# Check whether --with-boost was given. +if test "${with_boost+set}" = set; then + withval=$with_boost; + if test "$withval" = "no"; then + want_boost="no" + elif test "$withval" = "yes"; then + want_boost="yes" + ac_boost_path="" + else + want_boost="yes" + ac_boost_path="$withval" + fi + +else + want_boost="yes" +fi + + + + +# Check whether --with-boost-libdir was given. +if test "${with_boost_libdir+set}" = set; then + withval=$with_boost_libdir; + if test -d $withval + then + ac_boost_lib_path="$withval" + else + { { $as_echo "$as_me:$LINENO: error: --with-boost-libdir expected directory name" >&5 +$as_echo "$as_me: error: --with-boost-libdir expected directory name" >&2;} + { (exit 1); exit 1; }; } + fi + +else + ac_boost_lib_path="" + +fi + + +if test "x$want_boost" = "xyes"; then + boost_lib_version_req=1.35.0 + boost_lib_version_req_shorten=`expr $boost_lib_version_req : '\([0-9]*\.[0-9]*\)'` + boost_lib_version_req_major=`expr $boost_lib_version_req : '\([0-9]*\)'` + boost_lib_version_req_minor=`expr $boost_lib_version_req : '[0-9]*\.\([0-9]*\)'` + boost_lib_version_req_sub_minor=`expr $boost_lib_version_req : '[0-9]*\.[0-9]*\.\([0-9]*\)'` + if test "x$boost_lib_version_req_sub_minor" = "x" ; then + boost_lib_version_req_sub_minor="0" + fi + WANT_BOOST_VERSION=`expr $boost_lib_version_req_major \* 100000 \+ $boost_lib_version_req_minor \* 100 \+ $boost_lib_version_req_sub_minor` + { $as_echo "$as_me:$LINENO: checking for boostlib >= $boost_lib_version_req" >&5 +$as_echo_n "checking for boostlib >= $boost_lib_version_req... " >&6; } + succeeded=no + + if test "$ac_boost_path" != ""; then + BOOST_LDFLAGS="-L$ac_boost_path/lib" + BOOST_CPPFLAGS="-I$ac_boost_path/include" + else + for ac_boost_path_tmp in /usr /usr/local /opt /opt/local ; do + if test -d "$ac_boost_path_tmp/include/boost" && test -r "$ac_boost_path_tmp/include/boost"; then + BOOST_LDFLAGS="-L$ac_boost_path_tmp/lib" + BOOST_CPPFLAGS="-I$ac_boost_path_tmp/include" + break; + fi + done + fi + + if test "$ac_boost_lib_path" != ""; then + BOOST_LDFLAGS="-L$ac_boost_lib_path" + fi + + CPPFLAGS_SAVED="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + export CPPFLAGS + + LDFLAGS_SAVED="$LDFLAGS" + LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" + export LDFLAGS + + ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu + + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + + #include + +int +main () +{ + + #if BOOST_VERSION >= $WANT_BOOST_VERSION + // Everything is okay + #else + # error Boost version is too old + #endif + + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + + { $as_echo "$as_me:$LINENO: result: yes" >&5 +$as_echo "yes" >&6; } + succeeded=yes + found_system=yes + +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + + +fi + +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu + + + + + if test "x$succeeded" != "xyes"; then + _version=0 + if test "$ac_boost_path" != ""; then + if test -d "$ac_boost_path" && test -r "$ac_boost_path"; then + for i in `ls -d $ac_boost_path/include/boost-* 2>/dev/null`; do + _version_tmp=`echo $i | sed "s#$ac_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'` + V_CHECK=`expr $_version_tmp \> $_version` + if test "$V_CHECK" = "1" ; then + _version=$_version_tmp + fi + VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'` + BOOST_CPPFLAGS="-I$ac_boost_path/include/boost-$VERSION_UNDERSCORE" + done + fi + else + for ac_boost_path in /usr /usr/local /opt /opt/local ; do + if test -d "$ac_boost_path" && test -r "$ac_boost_path"; then + for i in `ls -d $ac_boost_path/include/boost-* 2>/dev/null`; do + _version_tmp=`echo $i | sed "s#$ac_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'` + V_CHECK=`expr $_version_tmp \> $_version` + if test "$V_CHECK" = "1" ; then + _version=$_version_tmp + best_path=$ac_boost_path + fi + done + fi + done + + VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'` + BOOST_CPPFLAGS="-I$best_path/include/boost-$VERSION_UNDERSCORE" + if test "$ac_boost_lib_path" = "" + then + BOOST_LDFLAGS="-L$best_path/lib" + fi + + if test "x$BOOST_ROOT" != "x"; then + if test -d "$BOOST_ROOT" && test -r "$BOOST_ROOT" && test -d "$BOOST_ROOT/stage/lib" && test -r "$BOOST_ROOT/stage/lib"; then + version_dir=`expr //$BOOST_ROOT : '.*/\(.*\)'` + stage_version=`echo $version_dir | sed 's/boost_//' | sed 's/_/./g'` + stage_version_shorten=`expr $stage_version : '\([0-9]*\.[0-9]*\)'` + V_CHECK=`expr $stage_version_shorten \>\= $_version` + if test "$V_CHECK" = "1" -a "$ac_boost_lib_path" = "" ; then + { $as_echo "$as_me:$LINENO: We will use a staged boost library from $BOOST_ROOT" >&5 +$as_echo "$as_me: We will use a staged boost library from $BOOST_ROOT" >&6;} + BOOST_CPPFLAGS="-I$BOOST_ROOT" + BOOST_LDFLAGS="-L$BOOST_ROOT/stage/lib" + fi + fi + fi + fi + + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + export CPPFLAGS + LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" + export LDFLAGS + + ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu + + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + + #include + +int +main () +{ + + #if BOOST_VERSION >= $WANT_BOOST_VERSION + // Everything is okay + #else + # error Boost version is too old + #endif + + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + + { $as_echo "$as_me:$LINENO: result: yes" >&5 +$as_echo "yes" >&6; } + succeeded=yes + found_system=yes + +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + + +fi + +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu + + fi + + if test "$succeeded" != "yes" ; then + if test "$_version" = "0" ; then + { { $as_echo "$as_me:$LINENO: error: We could not detect the boost libraries (version $boost_lib_version_req_shorten or higher). If you have a staged boost library (still not installed) please specify \$BOOST_ROOT in your environment and do not give a PATH to --with-boost option. If you are sure you have boost installed, then check your version number looking in . See http://randspringer.de/boost for more documentation." >&5 +$as_echo "$as_me: error: We could not detect the boost libraries (version $boost_lib_version_req_shorten or higher). If you have a staged boost library (still not installed) please specify \$BOOST_ROOT in your environment and do not give a PATH to --with-boost option. If you are sure you have boost installed, then check your version number looking in . See http://randspringer.de/boost for more documentation." >&2;} + { (exit 1); exit 1; }; } + else + { $as_echo "$as_me:$LINENO: Your boost libraries seems to old (version $_version)." >&5 +$as_echo "$as_me: Your boost libraries seems to old (version $_version)." >&6;} + fi + else + + + +cat >>confdefs.h <<\_ACEOF +#define HAVE_BOOST /**/ +_ACEOF + + fi + + CPPFLAGS="$CPPFLAGS_SAVED" + LDFLAGS="$LDFLAGS_SAVED" +fi + + + + +{ $as_echo "$as_me:$LINENO: checking for cos in -lm" >&5 +$as_echo_n "checking for cos in -lm... " >&6; } +if test "${ac_cv_lib_m_cos+set}" = set; then + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lm $LIBS" +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char cos (); +int +main () +{ +return cos (); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext conftest$ac_exeext +if { (ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_link") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest$ac_exeext && { + test "$cross_compiling" = yes || + $as_test_x conftest$ac_exeext + }; then + ac_cv_lib_m_cos=yes +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_cv_lib_m_cos=no +fi + +rm -rf conftest.dSYM +rm -f core conftest.err conftest.$ac_objext conftest_ipa8_conftest.oo \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:$LINENO: result: $ac_cv_lib_m_cos" >&5 +$as_echo "$ac_cv_lib_m_cos" >&6; } +if test "x$ac_cv_lib_m_cos" = x""yes; then + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBM 1 +_ACEOF + + LIBS="-lm $LIBS" + +fi + + +{ $as_echo "$as_me:$LINENO: checking for gzopen in -lz" >&5 +$as_echo_n "checking for gzopen in -lz... " >&6; } +if test "${ac_cv_lib_z_gzopen+set}" = set; then + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lz $LIBS" +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char gzopen (); +int +main () +{ +return gzopen (); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext conftest$ac_exeext +if { (ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_link") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest$ac_exeext && { + test "$cross_compiling" = yes || + $as_test_x conftest$ac_exeext + }; then + ac_cv_lib_z_gzopen=yes +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_cv_lib_z_gzopen=no +fi + +rm -rf conftest.dSYM +rm -f core conftest.err conftest.$ac_objext conftest_ipa8_conftest.oo \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:$LINENO: result: $ac_cv_lib_z_gzopen" >&5 +$as_echo "$ac_cv_lib_z_gzopen" >&6; } +if test "x$ac_cv_lib_z_gzopen" = x""yes; then + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBZ 1 +_ACEOF + + LIBS="-lz $LIBS" + +fi + + +have_gsl=yes + +{ $as_echo "$as_me:$LINENO: checking for cblas_dgemm in -lgslcblas" >&5 +$as_echo_n "checking for cblas_dgemm in -lgslcblas... " >&6; } +if test "${ac_cv_lib_gslcblas_cblas_dgemm+set}" = set; then + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lgslcblas $LIBS" +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char cblas_dgemm (); +int +main () +{ +return cblas_dgemm (); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext conftest$ac_exeext +if { (ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_link") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest$ac_exeext && { + test "$cross_compiling" = yes || + $as_test_x conftest$ac_exeext + }; then + ac_cv_lib_gslcblas_cblas_dgemm=yes +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_cv_lib_gslcblas_cblas_dgemm=no +fi + +rm -rf conftest.dSYM +rm -f core conftest.err conftest.$ac_objext conftest_ipa8_conftest.oo \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:$LINENO: result: $ac_cv_lib_gslcblas_cblas_dgemm" >&5 +$as_echo "$ac_cv_lib_gslcblas_cblas_dgemm" >&6; } +if test "x$ac_cv_lib_gslcblas_cblas_dgemm" = x""yes; then + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBGSLCBLAS 1 +_ACEOF + + LIBS="-lgslcblas $LIBS" + +else + have_gsl=no +fi + + +{ $as_echo "$as_me:$LINENO: checking for gsl_blas_dgemm in -lgsl" >&5 +$as_echo_n "checking for gsl_blas_dgemm in -lgsl... " >&6; } +if test "${ac_cv_lib_gsl_gsl_blas_dgemm+set}" = set; then + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lgsl $LIBS" +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char gsl_blas_dgemm (); +int +main () +{ +return gsl_blas_dgemm (); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext conftest$ac_exeext +if { (ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_link") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest$ac_exeext && { + test "$cross_compiling" = yes || + $as_test_x conftest$ac_exeext + }; then + ac_cv_lib_gsl_gsl_blas_dgemm=yes +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_cv_lib_gsl_gsl_blas_dgemm=no +fi + +rm -rf conftest.dSYM +rm -f core conftest.err conftest.$ac_objext conftest_ipa8_conftest.oo \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:$LINENO: result: $ac_cv_lib_gsl_gsl_blas_dgemm" >&5 +$as_echo "$ac_cv_lib_gsl_gsl_blas_dgemm" >&6; } +if test "x$ac_cv_lib_gsl_gsl_blas_dgemm" = x""yes; then + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBGSL 1 +_ACEOF + + LIBS="-lgsl $LIBS" + +else + have_gsl=no +fi + + +ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu +{ $as_echo "$as_me:$LINENO: checking how to run the C++ preprocessor" >&5 +$as_echo_n "checking how to run the C++ preprocessor... " >&6; } +if test -z "$CXXCPP"; then + if test "${ac_cv_prog_CXXCPP+set}" = set; then + $as_echo_n "(cached) " >&6 +else + # Double quotes because CXXCPP needs to be expanded + for CXXCPP in "$CXX -E" "/lib/cpp" + do + ac_preproc_ok=false +for ac_cxx_preproc_warn_flag in '' yes +do + # Use a header file that comes with gcc, so configuring glibc + # with a fresh cross-compiler works. + # Prefer to if __STDC__ is defined, since + # exists even on freestanding compilers. + # On the NeXT, cc -E runs the code through the compiler's parser, + # not just through cpp. "Syntax error" is here to catch this case. + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#ifdef __STDC__ +# include +#else +# include +#endif + Syntax error +_ACEOF +if { (ac_try="$ac_cpp conftest.$ac_ext" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_cpp conftest.$ac_ext") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } >/dev/null && { + test -z "$ac_cxx_preproc_warn_flag$ac_cxx_werror_flag" || + test ! -s conftest.err + }; then + : +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + # Broken: fails on valid input. +continue +fi + +rm -f conftest.err conftest.$ac_ext + + # OK, works on sane cases. Now check whether nonexistent headers + # can be detected and how. + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#include +_ACEOF +if { (ac_try="$ac_cpp conftest.$ac_ext" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_cpp conftest.$ac_ext") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } >/dev/null && { + test -z "$ac_cxx_preproc_warn_flag$ac_cxx_werror_flag" || + test ! -s conftest.err + }; then + # Broken: success on invalid input. +continue +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + # Passes both tests. +ac_preproc_ok=: +break +fi + +rm -f conftest.err conftest.$ac_ext + +done +# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. +rm -f conftest.err conftest.$ac_ext +if $ac_preproc_ok; then + break +fi + + done + ac_cv_prog_CXXCPP=$CXXCPP + +fi + CXXCPP=$ac_cv_prog_CXXCPP +else + ac_cv_prog_CXXCPP=$CXXCPP +fi +{ $as_echo "$as_me:$LINENO: result: $CXXCPP" >&5 +$as_echo "$CXXCPP" >&6; } +ac_preproc_ok=false +for ac_cxx_preproc_warn_flag in '' yes +do + # Use a header file that comes with gcc, so configuring glibc + # with a fresh cross-compiler works. + # Prefer to if __STDC__ is defined, since + # exists even on freestanding compilers. + # On the NeXT, cc -E runs the code through the compiler's parser, + # not just through cpp. "Syntax error" is here to catch this case. + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#ifdef __STDC__ +# include +#else +# include +#endif + Syntax error +_ACEOF +if { (ac_try="$ac_cpp conftest.$ac_ext" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_cpp conftest.$ac_ext") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } >/dev/null && { + test -z "$ac_cxx_preproc_warn_flag$ac_cxx_werror_flag" || + test ! -s conftest.err + }; then + : +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + # Broken: fails on valid input. +continue +fi + +rm -f conftest.err conftest.$ac_ext + + # OK, works on sane cases. Now check whether nonexistent headers + # can be detected and how. + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#include +_ACEOF +if { (ac_try="$ac_cpp conftest.$ac_ext" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_cpp conftest.$ac_ext") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } >/dev/null && { + test -z "$ac_cxx_preproc_warn_flag$ac_cxx_werror_flag" || + test ! -s conftest.err + }; then + # Broken: success on invalid input. +continue +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + # Passes both tests. +ac_preproc_ok=: +break +fi + +rm -f conftest.err conftest.$ac_ext + +done +# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. +rm -f conftest.err conftest.$ac_ext +if $ac_preproc_ok; then + : +else + { { $as_echo "$as_me:$LINENO: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +{ { $as_echo "$as_me:$LINENO: error: C++ preprocessor \"$CXXCPP\" fails sanity check +See \`config.log' for more details." >&5 +$as_echo "$as_me: error: C++ preprocessor \"$CXXCPP\" fails sanity check +See \`config.log' for more details." >&2;} + { (exit 1); exit 1; }; }; } +fi + +ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu + + +{ $as_echo "$as_me:$LINENO: checking for grep that handles long lines and -e" >&5 +$as_echo_n "checking for grep that handles long lines and -e... " >&6; } +if test "${ac_cv_path_GREP+set}" = set; then + $as_echo_n "(cached) " >&6 +else + if test -z "$GREP"; then + ac_path_GREP_found=false + # Loop through the user's path and test for each of PROGNAME-LIST + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH$PATH_SEPARATOR/usr/xpg4/bin +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_prog in grep ggrep; do + for ac_exec_ext in '' $ac_executable_extensions; do + ac_path_GREP="$as_dir/$ac_prog$ac_exec_ext" + { test -f "$ac_path_GREP" && $as_test_x "$ac_path_GREP"; } || continue +# Check for GNU ac_path_GREP and select it if it is found. + # Check for GNU $ac_path_GREP +case `"$ac_path_GREP" --version 2>&1` in +*GNU*) + ac_cv_path_GREP="$ac_path_GREP" ac_path_GREP_found=:;; +*) + ac_count=0 + $as_echo_n 0123456789 >"conftest.in" + while : + do + cat "conftest.in" "conftest.in" >"conftest.tmp" + mv "conftest.tmp" "conftest.in" + cp "conftest.in" "conftest.nl" + $as_echo 'GREP' >> "conftest.nl" + "$ac_path_GREP" -e 'GREP$' -e '-(cannot match)-' < "conftest.nl" >"conftest.out" 2>/dev/null || break + diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break + ac_count=`expr $ac_count + 1` + if test $ac_count -gt ${ac_path_GREP_max-0}; then + # Best one so far, save it but keep looking for a better one + ac_cv_path_GREP="$ac_path_GREP" + ac_path_GREP_max=$ac_count + fi + # 10*(2^10) chars as input seems more than enough + test $ac_count -gt 10 && break + done + rm -f conftest.in conftest.tmp conftest.nl conftest.out;; +esac + + $ac_path_GREP_found && break 3 + done + done +done +IFS=$as_save_IFS + if test -z "$ac_cv_path_GREP"; then + { { $as_echo "$as_me:$LINENO: error: no acceptable grep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" >&5 +$as_echo "$as_me: error: no acceptable grep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" >&2;} + { (exit 1); exit 1; }; } + fi +else + ac_cv_path_GREP=$GREP +fi + +fi +{ $as_echo "$as_me:$LINENO: result: $ac_cv_path_GREP" >&5 +$as_echo "$ac_cv_path_GREP" >&6; } + GREP="$ac_cv_path_GREP" + + +{ $as_echo "$as_me:$LINENO: checking for egrep" >&5 +$as_echo_n "checking for egrep... " >&6; } +if test "${ac_cv_path_EGREP+set}" = set; then + $as_echo_n "(cached) " >&6 +else + if echo a | $GREP -E '(a|b)' >/dev/null 2>&1 + then ac_cv_path_EGREP="$GREP -E" + else + if test -z "$EGREP"; then + ac_path_EGREP_found=false + # Loop through the user's path and test for each of PROGNAME-LIST + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH$PATH_SEPARATOR/usr/xpg4/bin +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_prog in egrep; do + for ac_exec_ext in '' $ac_executable_extensions; do + ac_path_EGREP="$as_dir/$ac_prog$ac_exec_ext" + { test -f "$ac_path_EGREP" && $as_test_x "$ac_path_EGREP"; } || continue +# Check for GNU ac_path_EGREP and select it if it is found. + # Check for GNU $ac_path_EGREP +case `"$ac_path_EGREP" --version 2>&1` in +*GNU*) + ac_cv_path_EGREP="$ac_path_EGREP" ac_path_EGREP_found=:;; +*) + ac_count=0 + $as_echo_n 0123456789 >"conftest.in" + while : + do + cat "conftest.in" "conftest.in" >"conftest.tmp" + mv "conftest.tmp" "conftest.in" + cp "conftest.in" "conftest.nl" + $as_echo 'EGREP' >> "conftest.nl" + "$ac_path_EGREP" 'EGREP$' < "conftest.nl" >"conftest.out" 2>/dev/null || break + diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break + ac_count=`expr $ac_count + 1` + if test $ac_count -gt ${ac_path_EGREP_max-0}; then + # Best one so far, save it but keep looking for a better one + ac_cv_path_EGREP="$ac_path_EGREP" + ac_path_EGREP_max=$ac_count + fi + # 10*(2^10) chars as input seems more than enough + test $ac_count -gt 10 && break + done + rm -f conftest.in conftest.tmp conftest.nl conftest.out;; +esac + + $ac_path_EGREP_found && break 3 + done + done +done +IFS=$as_save_IFS + if test -z "$ac_cv_path_EGREP"; then + { { $as_echo "$as_me:$LINENO: error: no acceptable egrep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" >&5 +$as_echo "$as_me: error: no acceptable egrep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" >&2;} + { (exit 1); exit 1; }; } + fi +else + ac_cv_path_EGREP=$EGREP +fi + + fi +fi +{ $as_echo "$as_me:$LINENO: result: $ac_cv_path_EGREP" >&5 +$as_echo "$ac_cv_path_EGREP" >&6; } + EGREP="$ac_cv_path_EGREP" + + +{ $as_echo "$as_me:$LINENO: checking for ANSI C header files" >&5 +$as_echo_n "checking for ANSI C header files... " >&6; } +if test "${ac_cv_header_stdc+set}" = set; then + $as_echo_n "(cached) " >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#include +#include +#include +#include + +int +main () +{ + + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + ac_cv_header_stdc=yes +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_cv_header_stdc=no +fi + +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + +if test $ac_cv_header_stdc = yes; then + # SunOS 4.x string.h does not declare mem*, contrary to ANSI. + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#include + +_ACEOF +if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | + $EGREP "memchr" >/dev/null 2>&1; then + : +else + ac_cv_header_stdc=no +fi +rm -f conftest* + +fi + +if test $ac_cv_header_stdc = yes; then + # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI. + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#include + +_ACEOF +if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | + $EGREP "free" >/dev/null 2>&1; then + : +else + ac_cv_header_stdc=no +fi +rm -f conftest* + +fi + +if test $ac_cv_header_stdc = yes; then + # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi. + if test "$cross_compiling" = yes; then + : +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#include +#include +#if ((' ' & 0x0FF) == 0x020) +# define ISLOWER(c) ('a' <= (c) && (c) <= 'z') +# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c)) +#else +# define ISLOWER(c) \ + (('a' <= (c) && (c) <= 'i') \ + || ('j' <= (c) && (c) <= 'r') \ + || ('s' <= (c) && (c) <= 'z')) +# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c)) +#endif + +#define XOR(e, f) (((e) && !(f)) || (!(e) && (f))) +int +main () +{ + int i; + for (i = 0; i < 256; i++) + if (XOR (islower (i), ISLOWER (i)) + || toupper (i) != TOUPPER (i)) + return 2; + return 0; +} +_ACEOF +rm -f conftest$ac_exeext +if { (ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_link") 2>&5 + ac_status=$? + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { ac_try='./conftest$ac_exeext' + { (case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_try") 2>&5 + ac_status=$? + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + : +else + $as_echo "$as_me: program exited with status $ac_status" >&5 +$as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +( exit $ac_status ) +ac_cv_header_stdc=no +fi +rm -rf conftest.dSYM +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext +fi + + +fi +fi +{ $as_echo "$as_me:$LINENO: result: $ac_cv_header_stdc" >&5 +$as_echo "$ac_cv_header_stdc" >&6; } +if test $ac_cv_header_stdc = yes; then + +cat >>confdefs.h <<\_ACEOF +#define STDC_HEADERS 1 +_ACEOF + +fi + +# On IRIX 5.3, sys/types and inttypes.h are conflicting. + + + + + + + + + +for ac_header in sys/types.h sys/stat.h stdlib.h string.h memory.h strings.h \ + inttypes.h stdint.h unistd.h +do +as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` +{ $as_echo "$as_me:$LINENO: checking for $ac_header" >&5 +$as_echo_n "checking for $ac_header... " >&6; } +if { as_var=$as_ac_Header; eval "test \"\${$as_var+set}\" = set"; }; then + $as_echo_n "(cached) " >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +$ac_includes_default + +#include <$ac_header> +_ACEOF +rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + eval "$as_ac_Header=yes" +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + eval "$as_ac_Header=no" +fi + +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +ac_res=`eval 'as_val=${'$as_ac_Header'} + $as_echo "$as_val"'` + { $as_echo "$as_me:$LINENO: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } +as_val=`eval 'as_val=${'$as_ac_Header'} + $as_echo "$as_val"'` + if test "x$as_val" = x""yes; then + cat >>confdefs.h <<_ACEOF +#define `$as_echo "HAVE_$ac_header" | $as_tr_cpp` 1 +_ACEOF + +fi + +done + + +if test x$with_irstlm = xcheck; then + if test "${ac_cv_header_n_gram_h+set}" = set; then + { $as_echo "$as_me:$LINENO: checking for n_gram.h" >&5 +$as_echo_n "checking for n_gram.h... " >&6; } +if test "${ac_cv_header_n_gram_h+set}" = set; then + $as_echo_n "(cached) " >&6 +fi +{ $as_echo "$as_me:$LINENO: result: $ac_cv_header_n_gram_h" >&5 +$as_echo "$ac_cv_header_n_gram_h" >&6; } +else + # Is the header compilable? +{ $as_echo "$as_me:$LINENO: checking n_gram.h usability" >&5 +$as_echo_n "checking n_gram.h usability... " >&6; } +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +$ac_includes_default +#include +_ACEOF +rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + ac_header_compiler=yes +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_header_compiler=no +fi + +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +{ $as_echo "$as_me:$LINENO: result: $ac_header_compiler" >&5 +$as_echo "$ac_header_compiler" >&6; } + +# Is the header present? +{ $as_echo "$as_me:$LINENO: checking n_gram.h presence" >&5 +$as_echo_n "checking n_gram.h presence... " >&6; } +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#include +_ACEOF +if { (ac_try="$ac_cpp conftest.$ac_ext" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_cpp conftest.$ac_ext") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } >/dev/null && { + test -z "$ac_cxx_preproc_warn_flag$ac_cxx_werror_flag" || + test ! -s conftest.err + }; then + ac_header_preproc=yes +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_header_preproc=no +fi + +rm -f conftest.err conftest.$ac_ext +{ $as_echo "$as_me:$LINENO: result: $ac_header_preproc" >&5 +$as_echo "$ac_header_preproc" >&6; } + +# So? What about this header? +case $ac_header_compiler:$ac_header_preproc:$ac_cxx_preproc_warn_flag in + yes:no: ) + { $as_echo "$as_me:$LINENO: WARNING: n_gram.h: accepted by the compiler, rejected by the preprocessor!" >&5 +$as_echo "$as_me: WARNING: n_gram.h: accepted by the compiler, rejected by the preprocessor!" >&2;} + { $as_echo "$as_me:$LINENO: WARNING: n_gram.h: proceeding with the compiler's result" >&5 +$as_echo "$as_me: WARNING: n_gram.h: proceeding with the compiler's result" >&2;} + ac_header_preproc=yes + ;; + no:yes:* ) + { $as_echo "$as_me:$LINENO: WARNING: n_gram.h: present but cannot be compiled" >&5 +$as_echo "$as_me: WARNING: n_gram.h: present but cannot be compiled" >&2;} + { $as_echo "$as_me:$LINENO: WARNING: n_gram.h: check for missing prerequisite headers?" >&5 +$as_echo "$as_me: WARNING: n_gram.h: check for missing prerequisite headers?" >&2;} + { $as_echo "$as_me:$LINENO: WARNING: n_gram.h: see the Autoconf documentation" >&5 +$as_echo "$as_me: WARNING: n_gram.h: see the Autoconf documentation" >&2;} + { $as_echo "$as_me:$LINENO: WARNING: n_gram.h: section \"Present But Cannot Be Compiled\"" >&5 +$as_echo "$as_me: WARNING: n_gram.h: section \"Present But Cannot Be Compiled\"" >&2;} + { $as_echo "$as_me:$LINENO: WARNING: n_gram.h: proceeding with the preprocessor's result" >&5 +$as_echo "$as_me: WARNING: n_gram.h: proceeding with the preprocessor's result" >&2;} + { $as_echo "$as_me:$LINENO: WARNING: n_gram.h: in the future, the compiler will take precedence" >&5 +$as_echo "$as_me: WARNING: n_gram.h: in the future, the compiler will take precedence" >&2;} + ( cat <<\_ASBOX +## ---------------------------------- ## +## Report this to hardmeier at fbk.eu ## +## ---------------------------------- ## +_ASBOX + ) | sed "s/^/$as_me: WARNING: /" >&2 + ;; +esac +{ $as_echo "$as_me:$LINENO: checking for n_gram.h" >&5 +$as_echo_n "checking for n_gram.h... " >&6; } +if test "${ac_cv_header_n_gram_h+set}" = set; then + $as_echo_n "(cached) " >&6 +else + ac_cv_header_n_gram_h=$ac_header_preproc +fi +{ $as_echo "$as_me:$LINENO: result: $ac_cv_header_n_gram_h" >&5 +$as_echo "$ac_cv_header_n_gram_h" >&6; } + +fi +if test "x$ac_cv_header_n_gram_h" = x""yes; then + +cat >>confdefs.h <<\_ACEOF +#define HAVE_IRSTLM /**/ +_ACEOF + +else + with_irstlm=no +fi + + + +else + SAVE_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS -I${with_irstlm}/include" + + if test "${ac_cv_header_n_gram_h+set}" = set; then + { $as_echo "$as_me:$LINENO: checking for n_gram.h" >&5 +$as_echo_n "checking for n_gram.h... " >&6; } +if test "${ac_cv_header_n_gram_h+set}" = set; then + $as_echo_n "(cached) " >&6 +fi +{ $as_echo "$as_me:$LINENO: result: $ac_cv_header_n_gram_h" >&5 +$as_echo "$ac_cv_header_n_gram_h" >&6; } +else + # Is the header compilable? +{ $as_echo "$as_me:$LINENO: checking n_gram.h usability" >&5 +$as_echo_n "checking n_gram.h usability... " >&6; } +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +$ac_includes_default +#include +_ACEOF +rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + ac_header_compiler=yes +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_header_compiler=no +fi + +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +{ $as_echo "$as_me:$LINENO: result: $ac_header_compiler" >&5 +$as_echo "$ac_header_compiler" >&6; } + +# Is the header present? +{ $as_echo "$as_me:$LINENO: checking n_gram.h presence" >&5 +$as_echo_n "checking n_gram.h presence... " >&6; } +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#include +_ACEOF +if { (ac_try="$ac_cpp conftest.$ac_ext" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_cpp conftest.$ac_ext") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } >/dev/null && { + test -z "$ac_cxx_preproc_warn_flag$ac_cxx_werror_flag" || + test ! -s conftest.err + }; then + ac_header_preproc=yes +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_header_preproc=no +fi + +rm -f conftest.err conftest.$ac_ext +{ $as_echo "$as_me:$LINENO: result: $ac_header_preproc" >&5 +$as_echo "$ac_header_preproc" >&6; } + +# So? What about this header? +case $ac_header_compiler:$ac_header_preproc:$ac_cxx_preproc_warn_flag in + yes:no: ) + { $as_echo "$as_me:$LINENO: WARNING: n_gram.h: accepted by the compiler, rejected by the preprocessor!" >&5 +$as_echo "$as_me: WARNING: n_gram.h: accepted by the compiler, rejected by the preprocessor!" >&2;} + { $as_echo "$as_me:$LINENO: WARNING: n_gram.h: proceeding with the compiler's result" >&5 +$as_echo "$as_me: WARNING: n_gram.h: proceeding with the compiler's result" >&2;} + ac_header_preproc=yes + ;; + no:yes:* ) + { $as_echo "$as_me:$LINENO: WARNING: n_gram.h: present but cannot be compiled" >&5 +$as_echo "$as_me: WARNING: n_gram.h: present but cannot be compiled" >&2;} + { $as_echo "$as_me:$LINENO: WARNING: n_gram.h: check for missing prerequisite headers?" >&5 +$as_echo "$as_me: WARNING: n_gram.h: check for missing prerequisite headers?" >&2;} + { $as_echo "$as_me:$LINENO: WARNING: n_gram.h: see the Autoconf documentation" >&5 +$as_echo "$as_me: WARNING: n_gram.h: see the Autoconf documentation" >&2;} + { $as_echo "$as_me:$LINENO: WARNING: n_gram.h: section \"Present But Cannot Be Compiled\"" >&5 +$as_echo "$as_me: WARNING: n_gram.h: section \"Present But Cannot Be Compiled\"" >&2;} + { $as_echo "$as_me:$LINENO: WARNING: n_gram.h: proceeding with the preprocessor's result" >&5 +$as_echo "$as_me: WARNING: n_gram.h: proceeding with the preprocessor's result" >&2;} + { $as_echo "$as_me:$LINENO: WARNING: n_gram.h: in the future, the compiler will take precedence" >&5 +$as_echo "$as_me: WARNING: n_gram.h: in the future, the compiler will take precedence" >&2;} + ( cat <<\_ASBOX +## ---------------------------------- ## +## Report this to hardmeier at fbk.eu ## +## ---------------------------------- ## +_ASBOX + ) | sed "s/^/$as_me: WARNING: /" >&2 + ;; +esac +{ $as_echo "$as_me:$LINENO: checking for n_gram.h" >&5 +$as_echo_n "checking for n_gram.h... " >&6; } +if test "${ac_cv_header_n_gram_h+set}" = set; then + $as_echo_n "(cached) " >&6 +else + ac_cv_header_n_gram_h=$ac_header_preproc +fi +{ $as_echo "$as_me:$LINENO: result: $ac_cv_header_n_gram_h" >&5 +$as_echo "$ac_cv_header_n_gram_h" >&6; } + +fi +if test "x$ac_cv_header_n_gram_h" = x""yes; then + +cat >>confdefs.h <<\_ACEOF +#define HAVE_IRSTLM /**/ +_ACEOF + +else + { { $as_echo "$as_me:$LINENO: error: Cannot find IRSTLM!" >&5 +$as_echo "$as_me: error: Cannot find IRSTLM!" >&2;} + { (exit 1); exit 1; }; } +fi + + + + MY_ARCH=`uname -m` + LIB_IRSTLM="-lirstlm" + LDFLAGS="$LDFLAGS -L${with_irstlm}/lib/${MY_ARCH}" + LIBS="$LIBS $LIB_IRSTLM" + FMTLIBS="$FMTLIBS libirstlm.a" + +fi + + + +if test x$with_irstlm != xno; then + IRSTLM_TRUE= + IRSTLM_FALSE='#' +else + IRSTLM_TRUE='#' + IRSTLM_FALSE= +fi + + +if test x$enable_channel = xyes; then + if test x$with_irstlm = xno || test x$have_gsl = xno; then + { { $as_echo "$as_me:$LINENO: error: The channel scorer needs both GSL and irstlm." >&5 +$as_echo "$as_me: error: The channel scorer needs both GSL and irstlm." >&2;} + { (exit 1); exit 1; }; } +fi + +fi + + +# Checks for header files. +#AC_CHECK_HEADERS([fenv.h sys/time.h]) + +# Checks for typedefs, structures, and compiler characteristics. +{ $as_echo "$as_me:$LINENO: checking for size_t" >&5 +$as_echo_n "checking for size_t... " >&6; } +if test "${ac_cv_type_size_t+set}" = set; then + $as_echo_n "(cached) " >&6 +else + ac_cv_type_size_t=no +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +$ac_includes_default +int +main () +{ +if (sizeof (size_t)) + return 0; + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +$ac_includes_default +int +main () +{ +if (sizeof ((size_t))) + return 0; + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + : +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_cv_type_size_t=yes +fi + +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + +fi + +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +{ $as_echo "$as_me:$LINENO: result: $ac_cv_type_size_t" >&5 +$as_echo "$ac_cv_type_size_t" >&6; } +if test "x$ac_cv_type_size_t" = x""yes; then + : +else + +cat >>confdefs.h <<_ACEOF +#define size_t unsigned int +_ACEOF + +fi + +{ $as_echo "$as_me:$LINENO: checking for ptrdiff_t" >&5 +$as_echo_n "checking for ptrdiff_t... " >&6; } +if test "${ac_cv_type_ptrdiff_t+set}" = set; then + $as_echo_n "(cached) " >&6 +else + ac_cv_type_ptrdiff_t=no +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +$ac_includes_default +int +main () +{ +if (sizeof (ptrdiff_t)) + return 0; + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +$ac_includes_default +int +main () +{ +if (sizeof ((ptrdiff_t))) + return 0; + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_compile") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then + : +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_cv_type_ptrdiff_t=yes +fi + +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + +fi + +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +{ $as_echo "$as_me:$LINENO: result: $ac_cv_type_ptrdiff_t" >&5 +$as_echo "$ac_cv_type_ptrdiff_t" >&6; } +if test "x$ac_cv_type_ptrdiff_t" = x""yes; then + +cat >>confdefs.h <<_ACEOF +#define HAVE_PTRDIFF_T 1 +_ACEOF + + +fi + + +# Checks for library functions. +#AC_FUNC_MALLOC +#AC_CHECK_FUNCS([getpagesize gettimeofday]) + + + +if test x$enable_channel = xyes; then + CHANNEL_SCORER_TRUE= + CHANNEL_SCORER_FALSE='#' +else + CHANNEL_SCORER_TRUE='#' + CHANNEL_SCORER_FALSE= +fi + + +ac_config_files="$ac_config_files Makefile" + +cat >confcache <<\_ACEOF +# This file is a shell script that caches the results of configure +# tests run on this system so they can be shared between configure +# scripts and configure runs, see configure's option --config-cache. +# It is not useful on other systems. If it contains results you don't +# want to keep, you may remove or edit it. +# +# config.status only pays attention to the cache file if you give it +# the --recheck option to rerun configure. +# +# `ac_cv_env_foo' variables (set or unset) will be overridden when +# loading this file, other *unset* `ac_cv_foo' will be assigned the +# following values. + +_ACEOF + +# The following way of writing the cache mishandles newlines in values, +# but we know of no workaround that is simple, portable, and efficient. +# So, we kill variables containing newlines. +# Ultrix sh set writes to stderr and can't be redirected directly, +# and sets the high bit in the cache file unless we assign to the vars. +( + for ac_var in `(set) 2>&1 | sed -n 's/^\([a-zA-Z_][a-zA-Z0-9_]*\)=.*/\1/p'`; do + eval ac_val=\$$ac_var + case $ac_val in #( + *${as_nl}*) + case $ac_var in #( + *_cv_*) { $as_echo "$as_me:$LINENO: WARNING: cache variable $ac_var contains a newline" >&5 +$as_echo "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;; + esac + case $ac_var in #( + _ | IFS | as_nl) ;; #( + BASH_ARGV | BASH_SOURCE) eval $ac_var= ;; #( + *) $as_unset $ac_var ;; + esac ;; + esac + done + + (set) 2>&1 | + case $as_nl`(ac_space=' '; set) 2>&1` in #( + *${as_nl}ac_space=\ *) + # `set' does not quote correctly, so add quotes (double-quote + # substitution turns \\\\ into \\, and sed turns \\ into \). + sed -n \ + "s/'/'\\\\''/g; + s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\\2'/p" + ;; #( + *) + # `set' quotes correctly as required by POSIX, so do not add quotes. + sed -n "/^[_$as_cr_alnum]*_cv_[_$as_cr_alnum]*=/p" + ;; + esac | + sort +) | + sed ' + /^ac_cv_env_/b end + t clear + :clear + s/^\([^=]*\)=\(.*[{}].*\)$/test "${\1+set}" = set || &/ + t end + s/^\([^=]*\)=\(.*\)$/\1=${\1=\2}/ + :end' >>confcache +if diff "$cache_file" confcache >/dev/null 2>&1; then :; else + if test -w "$cache_file"; then + test "x$cache_file" != "x/dev/null" && + { $as_echo "$as_me:$LINENO: updating cache $cache_file" >&5 +$as_echo "$as_me: updating cache $cache_file" >&6;} + cat confcache >$cache_file + else + { $as_echo "$as_me:$LINENO: not updating unwritable cache $cache_file" >&5 +$as_echo "$as_me: not updating unwritable cache $cache_file" >&6;} + fi +fi +rm -f confcache + +test "x$prefix" = xNONE && prefix=$ac_default_prefix +# Let make expand exec_prefix. +test "x$exec_prefix" = xNONE && exec_prefix='${prefix}' + +DEFS=-DHAVE_CONFIG_H + +ac_libobjs= +ac_ltlibobjs= +for ac_i in : $LIBOBJS; do test "x$ac_i" = x: && continue + # 1. Remove the extension, and $U if already installed. + ac_script='s/\$U\././;s/\.o$//;s/\.obj$//' + ac_i=`$as_echo "$ac_i" | sed "$ac_script"` + # 2. Prepend LIBOBJDIR. When used with automake>=1.10 LIBOBJDIR + # will be set to the directory where LIBOBJS objects are built. + ac_libobjs="$ac_libobjs \${LIBOBJDIR}$ac_i\$U.$ac_objext" + ac_ltlibobjs="$ac_ltlibobjs \${LIBOBJDIR}$ac_i"'$U.lo' +done +LIBOBJS=$ac_libobjs + +LTLIBOBJS=$ac_ltlibobjs + + +if test -z "${AMDEP_TRUE}" && test -z "${AMDEP_FALSE}"; then + { { $as_echo "$as_me:$LINENO: error: conditional \"AMDEP\" was never defined. +Usually this means the macro was only invoked conditionally." >&5 +$as_echo "$as_me: error: conditional \"AMDEP\" was never defined. +Usually this means the macro was only invoked conditionally." >&2;} + { (exit 1); exit 1; }; } +fi +if test -z "${am__fastdepCXX_TRUE}" && test -z "${am__fastdepCXX_FALSE}"; then + { { $as_echo "$as_me:$LINENO: error: conditional \"am__fastdepCXX\" was never defined. +Usually this means the macro was only invoked conditionally." >&5 +$as_echo "$as_me: error: conditional \"am__fastdepCXX\" was never defined. +Usually this means the macro was only invoked conditionally." >&2;} + { (exit 1); exit 1; }; } +fi +if test -z "${am__fastdepCC_TRUE}" && test -z "${am__fastdepCC_FALSE}"; then + { { $as_echo "$as_me:$LINENO: error: conditional \"am__fastdepCC\" was never defined. +Usually this means the macro was only invoked conditionally." >&5 +$as_echo "$as_me: error: conditional \"am__fastdepCC\" was never defined. +Usually this means the macro was only invoked conditionally." >&2;} + { (exit 1); exit 1; }; } +fi +if test -z "${IRSTLM_TRUE}" && test -z "${IRSTLM_FALSE}"; then + { { $as_echo "$as_me:$LINENO: error: conditional \"IRSTLM\" was never defined. +Usually this means the macro was only invoked conditionally." >&5 +$as_echo "$as_me: error: conditional \"IRSTLM\" was never defined. +Usually this means the macro was only invoked conditionally." >&2;} + { (exit 1); exit 1; }; } +fi +if test -z "${CHANNEL_SCORER_TRUE}" && test -z "${CHANNEL_SCORER_FALSE}"; then + { { $as_echo "$as_me:$LINENO: error: conditional \"CHANNEL_SCORER\" was never defined. +Usually this means the macro was only invoked conditionally." >&5 +$as_echo "$as_me: error: conditional \"CHANNEL_SCORER\" was never defined. +Usually this means the macro was only invoked conditionally." >&2;} + { (exit 1); exit 1; }; } +fi + +: ${CONFIG_STATUS=./config.status} +ac_write_fail=0 +ac_clean_files_save=$ac_clean_files +ac_clean_files="$ac_clean_files $CONFIG_STATUS" +{ $as_echo "$as_me:$LINENO: creating $CONFIG_STATUS" >&5 +$as_echo "$as_me: creating $CONFIG_STATUS" >&6;} +cat >$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +#! $SHELL +# Generated by $as_me. +# Run this file to recreate the current configuration. +# Compiler output produced by configure, useful for debugging +# configure, is in config.log if it exists. + +debug=false +ac_cs_recheck=false +ac_cs_silent=false +SHELL=\${CONFIG_SHELL-$SHELL} +_ACEOF + +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +## --------------------- ## +## M4sh Initialization. ## +## --------------------- ## + +# Be more Bourne compatible +DUALCASE=1; export DUALCASE # for MKS sh +if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then + emulate sh + NULLCMD=: + # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which + # is contrary to our usage. Disable this feature. + alias -g '${1+"$@"}'='"$@"' + setopt NO_GLOB_SUBST +else + case `(set -o) 2>/dev/null` in + *posix*) set -o posix ;; +esac + +fi + + + + +# PATH needs CR +# Avoid depending upon Character Ranges. +as_cr_letters='abcdefghijklmnopqrstuvwxyz' +as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ' +as_cr_Letters=$as_cr_letters$as_cr_LETTERS +as_cr_digits='0123456789' +as_cr_alnum=$as_cr_Letters$as_cr_digits + +as_nl=' +' +export as_nl +# Printing a long string crashes Solaris 7 /usr/bin/printf. +as_echo='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\' +as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo +as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo$as_echo +if (test "X`printf %s $as_echo`" = "X$as_echo") 2>/dev/null; then + as_echo='printf %s\n' + as_echo_n='printf %s' +else + if test "X`(/usr/ucb/echo -n -n $as_echo) 2>/dev/null`" = "X-n $as_echo"; then + as_echo_body='eval /usr/ucb/echo -n "$1$as_nl"' + as_echo_n='/usr/ucb/echo -n' + else + as_echo_body='eval expr "X$1" : "X\\(.*\\)"' + as_echo_n_body='eval + arg=$1; + case $arg in + *"$as_nl"*) + expr "X$arg" : "X\\(.*\\)$as_nl"; + arg=`expr "X$arg" : ".*$as_nl\\(.*\\)"`;; + esac; + expr "X$arg" : "X\\(.*\\)" | tr -d "$as_nl" + ' + export as_echo_n_body + as_echo_n='sh -c $as_echo_n_body as_echo' + fi + export as_echo_body + as_echo='sh -c $as_echo_body as_echo' +fi + +# The user is always right. +if test "${PATH_SEPARATOR+set}" != set; then + PATH_SEPARATOR=: + (PATH='/bin;/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 && { + (PATH='/bin:/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 || + PATH_SEPARATOR=';' + } +fi + +# Support unset when possible. +if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then + as_unset=unset +else + as_unset=false +fi + + +# IFS +# We need space, tab and new line, in precisely that order. Quoting is +# there to prevent editors from complaining about space-tab. +# (If _AS_PATH_WALK were called with IFS unset, it would disable word +# splitting by setting IFS to empty value.) +IFS=" "" $as_nl" + +# Find who we are. Look in the path if we contain no directory separator. +case $0 in + *[\\/]* ) as_myself=$0 ;; + *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break +done +IFS=$as_save_IFS + + ;; +esac +# We did not find ourselves, most probably we were run as `sh COMMAND' +# in which case we are not to be found in the path. +if test "x$as_myself" = x; then + as_myself=$0 +fi +if test ! -f "$as_myself"; then + $as_echo "$as_myself: error: cannot find myself; rerun with an absolute file name" >&2 + { (exit 1); exit 1; } +fi + +# Work around bugs in pre-3.0 UWIN ksh. +for as_var in ENV MAIL MAILPATH +do ($as_unset $as_var) >/dev/null 2>&1 && $as_unset $as_var +done +PS1='$ ' +PS2='> ' +PS4='+ ' + +# NLS nuisances. +LC_ALL=C +export LC_ALL +LANGUAGE=C +export LANGUAGE + +# Required to use basename. +if expr a : '\(a\)' >/dev/null 2>&1 && + test "X`expr 00001 : '.*\(...\)'`" = X001; then + as_expr=expr +else + as_expr=false +fi + +if (basename -- /) >/dev/null 2>&1 && test "X`basename -- / 2>&1`" = "X/"; then + as_basename=basename +else + as_basename=false +fi + + +# Name of the executable. +as_me=`$as_basename -- "$0" || +$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \ + X"$0" : 'X\(//\)$' \| \ + X"$0" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X/"$0" | + sed '/^.*\/\([^/][^/]*\)\/*$/{ + s//\1/ + q + } + /^X\/\(\/\/\)$/{ + s//\1/ + q + } + /^X\/\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'` + +# CDPATH. +$as_unset CDPATH + + + + as_lineno_1=$LINENO + as_lineno_2=$LINENO + test "x$as_lineno_1" != "x$as_lineno_2" && + test "x`expr $as_lineno_1 + 1`" = "x$as_lineno_2" || { + + # Create $as_me.lineno as a copy of $as_myself, but with $LINENO + # uniformly replaced by the line number. The first 'sed' inserts a + # line-number line after each line using $LINENO; the second 'sed' + # does the real work. The second script uses 'N' to pair each + # line-number line with the line containing $LINENO, and appends + # trailing '-' during substitution so that $LINENO is not a special + # case at line end. + # (Raja R Harinath suggested sed '=', and Paul Eggert wrote the + # scripts with optimization help from Paolo Bonzini. Blame Lee + # E. McMahon (1931-1989) for sed's syntax. :-) + sed -n ' + p + /[$]LINENO/= + ' <$as_myself | + sed ' + s/[$]LINENO.*/&-/ + t lineno + b + :lineno + N + :loop + s/[$]LINENO\([^'$as_cr_alnum'_].*\n\)\(.*\)/\2\1\2/ + t loop + s/-\n.*// + ' >$as_me.lineno && + chmod +x "$as_me.lineno" || + { $as_echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2 + { (exit 1); exit 1; }; } + + # Don't try to exec as it changes $[0], causing all sort of problems + # (the dirname of $[0] is not the place where we might find the + # original and so on. Autoconf is especially sensitive to this). + . "./$as_me.lineno" + # Exit status is that of the last command. + exit +} + + +if (as_dir=`dirname -- /` && test "X$as_dir" = X/) >/dev/null 2>&1; then + as_dirname=dirname +else + as_dirname=false +fi + +ECHO_C= ECHO_N= ECHO_T= +case `echo -n x` in +-n*) + case `echo 'x\c'` in + *c*) ECHO_T=' ';; # ECHO_T is single tab character. + *) ECHO_C='\c';; + esac;; +*) + ECHO_N='-n';; +esac +if expr a : '\(a\)' >/dev/null 2>&1 && + test "X`expr 00001 : '.*\(...\)'`" = X001; then + as_expr=expr +else + as_expr=false +fi + +rm -f conf$$ conf$$.exe conf$$.file +if test -d conf$$.dir; then + rm -f conf$$.dir/conf$$.file +else + rm -f conf$$.dir + mkdir conf$$.dir 2>/dev/null +fi +if (echo >conf$$.file) 2>/dev/null; then + if ln -s conf$$.file conf$$ 2>/dev/null; then + as_ln_s='ln -s' + # ... but there are two gotchas: + # 1) On MSYS, both `ln -s file dir' and `ln file dir' fail. + # 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable. + # In both cases, we have to default to `cp -p'. + ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe || + as_ln_s='cp -p' + elif ln conf$$.file conf$$ 2>/dev/null; then + as_ln_s=ln + else + as_ln_s='cp -p' + fi +else + as_ln_s='cp -p' +fi +rm -f conf$$ conf$$.exe conf$$.dir/conf$$.file conf$$.file +rmdir conf$$.dir 2>/dev/null + +if mkdir -p . 2>/dev/null; then + as_mkdir_p=: +else + test -d ./-p && rmdir ./-p + as_mkdir_p=false +fi + +if test -x / >/dev/null 2>&1; then + as_test_x='test -x' +else + if ls -dL / >/dev/null 2>&1; then + as_ls_L_option=L + else + as_ls_L_option= + fi + as_test_x=' + eval sh -c '\'' + if test -d "$1"; then + test -d "$1/."; + else + case $1 in + -*)set "./$1";; + esac; + case `ls -ld'$as_ls_L_option' "$1" 2>/dev/null` in + ???[sx]*):;;*)false;;esac;fi + '\'' sh + ' +fi +as_executable_p=$as_test_x + +# Sed expression to map a string onto a valid CPP name. +as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'" + +# Sed expression to map a string onto a valid variable name. +as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'" + + +exec 6>&1 + +# Save the log message, to keep $[0] and so on meaningful, and to +# report actual input values of CONFIG_FILES etc. instead of their +# values after options handling. +ac_log=" +This file was extended by memscore $as_me 1.0, which was +generated by GNU Autoconf 2.63. Invocation command line was + + CONFIG_FILES = $CONFIG_FILES + CONFIG_HEADERS = $CONFIG_HEADERS + CONFIG_LINKS = $CONFIG_LINKS + CONFIG_COMMANDS = $CONFIG_COMMANDS + $ $0 $@ + +on `(hostname || uname -n) 2>/dev/null | sed 1q` +" + +_ACEOF + +case $ac_config_files in *" +"*) set x $ac_config_files; shift; ac_config_files=$*;; +esac + +case $ac_config_headers in *" +"*) set x $ac_config_headers; shift; ac_config_headers=$*;; +esac + + +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +# Files that config.status was made for. +config_files="$ac_config_files" +config_headers="$ac_config_headers" +config_commands="$ac_config_commands" + +_ACEOF + +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +ac_cs_usage="\ +\`$as_me' instantiates files from templates according to the +current configuration. + +Usage: $0 [OPTION]... [FILE]... + + -h, --help print this help, then exit + -V, --version print version number and configuration settings, then exit + -q, --quiet, --silent + do not print progress messages + -d, --debug don't remove temporary files + --recheck update $as_me by reconfiguring in the same conditions + --file=FILE[:TEMPLATE] + instantiate the configuration file FILE + --header=FILE[:TEMPLATE] + instantiate the configuration header FILE + +Configuration files: +$config_files + +Configuration headers: +$config_headers + +Configuration commands: +$config_commands + +Report bugs to ." + +_ACEOF +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +ac_cs_version="\\ +memscore config.status 1.0 +configured by $0, generated by GNU Autoconf 2.63, + with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" + +Copyright (C) 2008 Free Software Foundation, Inc. +This config.status script is free software; the Free Software Foundation +gives unlimited permission to copy, distribute and modify it." + +ac_pwd='$ac_pwd' +srcdir='$srcdir' +INSTALL='$INSTALL' +AWK='$AWK' +test -n "\$AWK" || AWK=awk +_ACEOF + +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +# The default lists apply if the user does not specify any file. +ac_need_defaults=: +while test $# != 0 +do + case $1 in + --*=*) + ac_option=`expr "X$1" : 'X\([^=]*\)='` + ac_optarg=`expr "X$1" : 'X[^=]*=\(.*\)'` + ac_shift=: + ;; + *) + ac_option=$1 + ac_optarg=$2 + ac_shift=shift + ;; + esac + + case $ac_option in + # Handling of the options. + -recheck | --recheck | --rechec | --reche | --rech | --rec | --re | --r) + ac_cs_recheck=: ;; + --version | --versio | --versi | --vers | --ver | --ve | --v | -V ) + $as_echo "$ac_cs_version"; exit ;; + --debug | --debu | --deb | --de | --d | -d ) + debug=: ;; + --file | --fil | --fi | --f ) + $ac_shift + case $ac_optarg in + *\'*) ac_optarg=`$as_echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"` ;; + esac + CONFIG_FILES="$CONFIG_FILES '$ac_optarg'" + ac_need_defaults=false;; + --header | --heade | --head | --hea ) + $ac_shift + case $ac_optarg in + *\'*) ac_optarg=`$as_echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"` ;; + esac + CONFIG_HEADERS="$CONFIG_HEADERS '$ac_optarg'" + ac_need_defaults=false;; + --he | --h) + # Conflict between --help and --header + { $as_echo "$as_me: error: ambiguous option: $1 +Try \`$0 --help' for more information." >&2 + { (exit 1); exit 1; }; };; + --help | --hel | -h ) + $as_echo "$ac_cs_usage"; exit ;; + -q | -quiet | --quiet | --quie | --qui | --qu | --q \ + | -silent | --silent | --silen | --sile | --sil | --si | --s) + ac_cs_silent=: ;; + + # This is an error. + -*) { $as_echo "$as_me: error: unrecognized option: $1 +Try \`$0 --help' for more information." >&2 + { (exit 1); exit 1; }; } ;; + + *) ac_config_targets="$ac_config_targets $1" + ac_need_defaults=false ;; + + esac + shift +done + +ac_configure_extra_args= + +if $ac_cs_silent; then + exec 6>/dev/null + ac_configure_extra_args="$ac_configure_extra_args --silent" +fi + +_ACEOF +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +if \$ac_cs_recheck; then + set X '$SHELL' '$0' $ac_configure_args \$ac_configure_extra_args --no-create --no-recursion + shift + \$as_echo "running CONFIG_SHELL=$SHELL \$*" >&6 + CONFIG_SHELL='$SHELL' + export CONFIG_SHELL + exec "\$@" +fi + +_ACEOF +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +exec 5>>config.log +{ + echo + sed 'h;s/./-/g;s/^.../## /;s/...$/ ##/;p;x;p;x' <<_ASBOX +## Running $as_me. ## +_ASBOX + $as_echo "$ac_log" +} >&5 + +_ACEOF +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +# +# INIT-COMMANDS +# +AMDEP_TRUE="$AMDEP_TRUE" ac_aux_dir="$ac_aux_dir" + +_ACEOF + +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 + +# Handling of arguments. +for ac_config_target in $ac_config_targets +do + case $ac_config_target in + "config.h") CONFIG_HEADERS="$CONFIG_HEADERS config.h" ;; + "depfiles") CONFIG_COMMANDS="$CONFIG_COMMANDS depfiles" ;; + "Makefile") CONFIG_FILES="$CONFIG_FILES Makefile" ;; + + *) { { $as_echo "$as_me:$LINENO: error: invalid argument: $ac_config_target" >&5 +$as_echo "$as_me: error: invalid argument: $ac_config_target" >&2;} + { (exit 1); exit 1; }; };; + esac +done + + +# If the user did not use the arguments to specify the items to instantiate, +# then the envvar interface is used. Set only those that are not. +# We use the long form for the default assignment because of an extremely +# bizarre bug on SunOS 4.1.3. +if $ac_need_defaults; then + test "${CONFIG_FILES+set}" = set || CONFIG_FILES=$config_files + test "${CONFIG_HEADERS+set}" = set || CONFIG_HEADERS=$config_headers + test "${CONFIG_COMMANDS+set}" = set || CONFIG_COMMANDS=$config_commands +fi + +# Have a temporary directory for convenience. Make it in the build tree +# simply because there is no reason against having it here, and in addition, +# creating and moving files from /tmp can sometimes cause problems. +# Hook for its removal unless debugging. +# Note that there is a small window in which the directory will not be cleaned: +# after its creation but before its name has been assigned to `$tmp'. +$debug || +{ + tmp= + trap 'exit_status=$? + { test -z "$tmp" || test ! -d "$tmp" || rm -fr "$tmp"; } && exit $exit_status +' 0 + trap '{ (exit 1); exit 1; }' 1 2 13 15 +} +# Create a (secure) tmp directory for tmp files. + +{ + tmp=`(umask 077 && mktemp -d "./confXXXXXX") 2>/dev/null` && + test -n "$tmp" && test -d "$tmp" +} || +{ + tmp=./conf$$-$RANDOM + (umask 077 && mkdir "$tmp") +} || +{ + $as_echo "$as_me: cannot create a temporary directory in ." >&2 + { (exit 1); exit 1; } +} + +# Set up the scripts for CONFIG_FILES section. +# No need to generate them if there are no CONFIG_FILES. +# This happens for instance with `./config.status config.h'. +if test -n "$CONFIG_FILES"; then + + +ac_cr=' ' +ac_cs_awk_cr=`$AWK 'BEGIN { print "a\rb" }' /dev/null` +if test "$ac_cs_awk_cr" = "a${ac_cr}b"; then + ac_cs_awk_cr='\\r' +else + ac_cs_awk_cr=$ac_cr +fi + +echo 'BEGIN {' >"$tmp/subs1.awk" && +_ACEOF + + +{ + echo "cat >conf$$subs.awk <<_ACEOF" && + echo "$ac_subst_vars" | sed 's/.*/&!$&$ac_delim/' && + echo "_ACEOF" +} >conf$$subs.sh || + { { $as_echo "$as_me:$LINENO: error: could not make $CONFIG_STATUS" >&5 +$as_echo "$as_me: error: could not make $CONFIG_STATUS" >&2;} + { (exit 1); exit 1; }; } +ac_delim_num=`echo "$ac_subst_vars" | grep -c '$'` +ac_delim='%!_!# ' +for ac_last_try in false false false false false :; do + . ./conf$$subs.sh || + { { $as_echo "$as_me:$LINENO: error: could not make $CONFIG_STATUS" >&5 +$as_echo "$as_me: error: could not make $CONFIG_STATUS" >&2;} + { (exit 1); exit 1; }; } + + ac_delim_n=`sed -n "s/.*$ac_delim\$/X/p" conf$$subs.awk | grep -c X` + if test $ac_delim_n = $ac_delim_num; then + break + elif $ac_last_try; then + { { $as_echo "$as_me:$LINENO: error: could not make $CONFIG_STATUS" >&5 +$as_echo "$as_me: error: could not make $CONFIG_STATUS" >&2;} + { (exit 1); exit 1; }; } + else + ac_delim="$ac_delim!$ac_delim _$ac_delim!! " + fi +done +rm -f conf$$subs.sh + +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +cat >>"\$tmp/subs1.awk" <<\\_ACAWK && +_ACEOF +sed -n ' +h +s/^/S["/; s/!.*/"]=/ +p +g +s/^[^!]*!// +:repl +t repl +s/'"$ac_delim"'$// +t delim +:nl +h +s/\(.\{148\}\).*/\1/ +t more1 +s/["\\]/\\&/g; s/^/"/; s/$/\\n"\\/ +p +n +b repl +:more1 +s/["\\]/\\&/g; s/^/"/; s/$/"\\/ +p +g +s/.\{148\}// +t nl +:delim +h +s/\(.\{148\}\).*/\1/ +t more2 +s/["\\]/\\&/g; s/^/"/; s/$/"/ +p +b +:more2 +s/["\\]/\\&/g; s/^/"/; s/$/"\\/ +p +g +s/.\{148\}// +t delim +' >$CONFIG_STATUS || ac_write_fail=1 +rm -f conf$$subs.awk +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +_ACAWK +cat >>"\$tmp/subs1.awk" <<_ACAWK && + for (key in S) S_is_set[key] = 1 + FS = "" + +} +{ + line = $ 0 + nfields = split(line, field, "@") + substed = 0 + len = length(field[1]) + for (i = 2; i < nfields; i++) { + key = field[i] + keylen = length(key) + if (S_is_set[key]) { + value = S[key] + line = substr(line, 1, len) "" value "" substr(line, len + keylen + 3) + len += length(value) + length(field[++i]) + substed = 1 + } else + len += 1 + keylen + } + + print line +} + +_ACAWK +_ACEOF +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +if sed "s/$ac_cr//" < /dev/null > /dev/null 2>&1; then + sed "s/$ac_cr\$//; s/$ac_cr/$ac_cs_awk_cr/g" +else + cat +fi < "$tmp/subs1.awk" > "$tmp/subs.awk" \ + || { { $as_echo "$as_me:$LINENO: error: could not setup config files machinery" >&5 +$as_echo "$as_me: error: could not setup config files machinery" >&2;} + { (exit 1); exit 1; }; } +_ACEOF + +# VPATH may cause trouble with some makes, so we remove $(srcdir), +# ${srcdir} and @srcdir@ from VPATH if srcdir is ".", strip leading and +# trailing colons and then remove the whole line if VPATH becomes empty +# (actually we leave an empty line to preserve line numbers). +if test "x$srcdir" = x.; then + ac_vpsub='/^[ ]*VPATH[ ]*=/{ +s/:*\$(srcdir):*/:/ +s/:*\${srcdir}:*/:/ +s/:*@srcdir@:*/:/ +s/^\([^=]*=[ ]*\):*/\1/ +s/:*$// +s/^[^=]*=[ ]*$// +}' +fi + +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +fi # test -n "$CONFIG_FILES" + +# Set up the scripts for CONFIG_HEADERS section. +# No need to generate them if there are no CONFIG_HEADERS. +# This happens for instance with `./config.status Makefile'. +if test -n "$CONFIG_HEADERS"; then +cat >"$tmp/defines.awk" <<\_ACAWK || +BEGIN { +_ACEOF + +# Transform confdefs.h into an awk script `defines.awk', embedded as +# here-document in config.status, that substitutes the proper values into +# config.h.in to produce config.h. + +# Create a delimiter string that does not exist in confdefs.h, to ease +# handling of long lines. +ac_delim='%!_!# ' +for ac_last_try in false false :; do + ac_t=`sed -n "/$ac_delim/p" confdefs.h` + if test -z "$ac_t"; then + break + elif $ac_last_try; then + { { $as_echo "$as_me:$LINENO: error: could not make $CONFIG_HEADERS" >&5 +$as_echo "$as_me: error: could not make $CONFIG_HEADERS" >&2;} + { (exit 1); exit 1; }; } + else + ac_delim="$ac_delim!$ac_delim _$ac_delim!! " + fi +done + +# For the awk script, D is an array of macro values keyed by name, +# likewise P contains macro parameters if any. Preserve backslash +# newline sequences. + +ac_word_re=[_$as_cr_Letters][_$as_cr_alnum]* +sed -n ' +s/.\{148\}/&'"$ac_delim"'/g +t rset +:rset +s/^[ ]*#[ ]*define[ ][ ]*/ / +t def +d +:def +s/\\$// +t bsnl +s/["\\]/\\&/g +s/^ \('"$ac_word_re"'\)\(([^()]*)\)[ ]*\(.*\)/P["\1"]="\2"\ +D["\1"]=" \3"/p +s/^ \('"$ac_word_re"'\)[ ]*\(.*\)/D["\1"]=" \2"/p +d +:bsnl +s/["\\]/\\&/g +s/^ \('"$ac_word_re"'\)\(([^()]*)\)[ ]*\(.*\)/P["\1"]="\2"\ +D["\1"]=" \3\\\\\\n"\\/p +t cont +s/^ \('"$ac_word_re"'\)[ ]*\(.*\)/D["\1"]=" \2\\\\\\n"\\/p +t cont +d +:cont +n +s/.\{148\}/&'"$ac_delim"'/g +t clear +:clear +s/\\$// +t bsnlc +s/["\\]/\\&/g; s/^/"/; s/$/"/p +d +:bsnlc +s/["\\]/\\&/g; s/^/"/; s/$/\\\\\\n"\\/p +b cont +' >$CONFIG_STATUS || ac_write_fail=1 + +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 + for (key in D) D_is_set[key] = 1 + FS = "" +} +/^[\t ]*#[\t ]*(define|undef)[\t ]+$ac_word_re([\t (]|\$)/ { + line = \$ 0 + split(line, arg, " ") + if (arg[1] == "#") { + defundef = arg[2] + mac1 = arg[3] + } else { + defundef = substr(arg[1], 2) + mac1 = arg[2] + } + split(mac1, mac2, "(") #) + macro = mac2[1] + prefix = substr(line, 1, index(line, defundef) - 1) + if (D_is_set[macro]) { + # Preserve the white space surrounding the "#". + print prefix "define", macro P[macro] D[macro] + next + } else { + # Replace #undef with comments. This is necessary, for example, + # in the case of _POSIX_SOURCE, which is predefined and required + # on some systems where configure will not decide to define it. + if (defundef == "undef") { + print "/*", prefix defundef, macro, "*/" + next + } + } +} +{ print } +_ACAWK +_ACEOF +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 + { { $as_echo "$as_me:$LINENO: error: could not setup config headers machinery" >&5 +$as_echo "$as_me: error: could not setup config headers machinery" >&2;} + { (exit 1); exit 1; }; } +fi # test -n "$CONFIG_HEADERS" + + +eval set X " :F $CONFIG_FILES :H $CONFIG_HEADERS :C $CONFIG_COMMANDS" +shift +for ac_tag +do + case $ac_tag in + :[FHLC]) ac_mode=$ac_tag; continue;; + esac + case $ac_mode$ac_tag in + :[FHL]*:*);; + :L* | :C*:*) { { $as_echo "$as_me:$LINENO: error: invalid tag $ac_tag" >&5 +$as_echo "$as_me: error: invalid tag $ac_tag" >&2;} + { (exit 1); exit 1; }; };; + :[FH]-) ac_tag=-:-;; + :[FH]*) ac_tag=$ac_tag:$ac_tag.in;; + esac + ac_save_IFS=$IFS + IFS=: + set x $ac_tag + IFS=$ac_save_IFS + shift + ac_file=$1 + shift + + case $ac_mode in + :L) ac_source=$1;; + :[FH]) + ac_file_inputs= + for ac_f + do + case $ac_f in + -) ac_f="$tmp/stdin";; + *) # Look for the file first in the build tree, then in the source tree + # (if the path is not absolute). The absolute path cannot be DOS-style, + # because $ac_f cannot contain `:'. + test -f "$ac_f" || + case $ac_f in + [\\/$]*) false;; + *) test -f "$srcdir/$ac_f" && ac_f="$srcdir/$ac_f";; + esac || + { { $as_echo "$as_me:$LINENO: error: cannot find input file: $ac_f" >&5 +$as_echo "$as_me: error: cannot find input file: $ac_f" >&2;} + { (exit 1); exit 1; }; };; + esac + case $ac_f in *\'*) ac_f=`$as_echo "$ac_f" | sed "s/'/'\\\\\\\\''/g"`;; esac + ac_file_inputs="$ac_file_inputs '$ac_f'" + done + + # Let's still pretend it is `configure' which instantiates (i.e., don't + # use $as_me), people would be surprised to read: + # /* config.h. Generated by config.status. */ + configure_input='Generated from '` + $as_echo "$*" | sed 's|^[^:]*/||;s|:[^:]*/|, |g' + `' by configure.' + if test x"$ac_file" != x-; then + configure_input="$ac_file. $configure_input" + { $as_echo "$as_me:$LINENO: creating $ac_file" >&5 +$as_echo "$as_me: creating $ac_file" >&6;} + fi + # Neutralize special characters interpreted by sed in replacement strings. + case $configure_input in #( + *\&* | *\|* | *\\* ) + ac_sed_conf_input=`$as_echo "$configure_input" | + sed 's/[\\\\&|]/\\\\&/g'`;; #( + *) ac_sed_conf_input=$configure_input;; + esac + + case $ac_tag in + *:-:* | *:-) cat >"$tmp/stdin" \ + || { { $as_echo "$as_me:$LINENO: error: could not create $ac_file" >&5 +$as_echo "$as_me: error: could not create $ac_file" >&2;} + { (exit 1); exit 1; }; } ;; + esac + ;; + esac + + ac_dir=`$as_dirname -- "$ac_file" || +$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$ac_file" : 'X\(//\)[^/]' \| \ + X"$ac_file" : 'X\(//\)$' \| \ + X"$ac_file" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X"$ac_file" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'` + { as_dir="$ac_dir" + case $as_dir in #( + -*) as_dir=./$as_dir;; + esac + test -d "$as_dir" || { $as_mkdir_p && mkdir -p "$as_dir"; } || { + as_dirs= + while :; do + case $as_dir in #( + *\'*) as_qdir=`$as_echo "$as_dir" | sed "s/'/'\\\\\\\\''/g"`;; #'( + *) as_qdir=$as_dir;; + esac + as_dirs="'$as_qdir' $as_dirs" + as_dir=`$as_dirname -- "$as_dir" || +$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$as_dir" : 'X\(//\)[^/]' \| \ + X"$as_dir" : 'X\(//\)$' \| \ + X"$as_dir" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X"$as_dir" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'` + test -d "$as_dir" && break + done + test -z "$as_dirs" || eval "mkdir $as_dirs" + } || test -d "$as_dir" || { { $as_echo "$as_me:$LINENO: error: cannot create directory $as_dir" >&5 +$as_echo "$as_me: error: cannot create directory $as_dir" >&2;} + { (exit 1); exit 1; }; }; } + ac_builddir=. + +case "$ac_dir" in +.) ac_dir_suffix= ac_top_builddir_sub=. ac_top_build_prefix= ;; +*) + ac_dir_suffix=/`$as_echo "$ac_dir" | sed 's|^\.[\\/]||'` + # A ".." for each directory in $ac_dir_suffix. + ac_top_builddir_sub=`$as_echo "$ac_dir_suffix" | sed 's|/[^\\/]*|/..|g;s|/||'` + case $ac_top_builddir_sub in + "") ac_top_builddir_sub=. ac_top_build_prefix= ;; + *) ac_top_build_prefix=$ac_top_builddir_sub/ ;; + esac ;; +esac +ac_abs_top_builddir=$ac_pwd +ac_abs_builddir=$ac_pwd$ac_dir_suffix +# for backward compatibility: +ac_top_builddir=$ac_top_build_prefix + +case $srcdir in + .) # We are building in place. + ac_srcdir=. + ac_top_srcdir=$ac_top_builddir_sub + ac_abs_top_srcdir=$ac_pwd ;; + [\\/]* | ?:[\\/]* ) # Absolute name. + ac_srcdir=$srcdir$ac_dir_suffix; + ac_top_srcdir=$srcdir + ac_abs_top_srcdir=$srcdir ;; + *) # Relative name. + ac_srcdir=$ac_top_build_prefix$srcdir$ac_dir_suffix + ac_top_srcdir=$ac_top_build_prefix$srcdir + ac_abs_top_srcdir=$ac_pwd/$srcdir ;; +esac +ac_abs_srcdir=$ac_abs_top_srcdir$ac_dir_suffix + + + case $ac_mode in + :F) + # + # CONFIG_FILE + # + + case $INSTALL in + [\\/$]* | ?:[\\/]* ) ac_INSTALL=$INSTALL ;; + *) ac_INSTALL=$ac_top_build_prefix$INSTALL ;; + esac +_ACEOF + +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +# If the template does not know about datarootdir, expand it. +# FIXME: This hack should be removed a few years after 2.60. +ac_datarootdir_hack=; ac_datarootdir_seen= + +ac_sed_dataroot=' +/datarootdir/ { + p + q +} +/@datadir@/p +/@docdir@/p +/@infodir@/p +/@localedir@/p +/@mandir@/p +' +case `eval "sed -n \"\$ac_sed_dataroot\" $ac_file_inputs"` in +*datarootdir*) ac_datarootdir_seen=yes;; +*@datadir@*|*@docdir@*|*@infodir@*|*@localedir@*|*@mandir@*) + { $as_echo "$as_me:$LINENO: WARNING: $ac_file_inputs seems to ignore the --datarootdir setting" >&5 +$as_echo "$as_me: WARNING: $ac_file_inputs seems to ignore the --datarootdir setting" >&2;} +_ACEOF +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 + ac_datarootdir_hack=' + s&@datadir@&$datadir&g + s&@docdir@&$docdir&g + s&@infodir@&$infodir&g + s&@localedir@&$localedir&g + s&@mandir@&$mandir&g + s&\\\${datarootdir}&$datarootdir&g' ;; +esac +_ACEOF + +# Neutralize VPATH when `$srcdir' = `.'. +# Shell code in configure.ac might set extrasub. +# FIXME: do we really want to maintain this feature? +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +ac_sed_extra="$ac_vpsub +$extrasub +_ACEOF +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +:t +/@[a-zA-Z_][a-zA-Z_0-9]*@/!b +s|@configure_input@|$ac_sed_conf_input|;t t +s&@top_builddir@&$ac_top_builddir_sub&;t t +s&@top_build_prefix@&$ac_top_build_prefix&;t t +s&@srcdir@&$ac_srcdir&;t t +s&@abs_srcdir@&$ac_abs_srcdir&;t t +s&@top_srcdir@&$ac_top_srcdir&;t t +s&@abs_top_srcdir@&$ac_abs_top_srcdir&;t t +s&@builddir@&$ac_builddir&;t t +s&@abs_builddir@&$ac_abs_builddir&;t t +s&@abs_top_builddir@&$ac_abs_top_builddir&;t t +s&@INSTALL@&$ac_INSTALL&;t t +$ac_datarootdir_hack +" +eval sed \"\$ac_sed_extra\" "$ac_file_inputs" | $AWK -f "$tmp/subs.awk" >$tmp/out \ + || { { $as_echo "$as_me:$LINENO: error: could not create $ac_file" >&5 +$as_echo "$as_me: error: could not create $ac_file" >&2;} + { (exit 1); exit 1; }; } + +test -z "$ac_datarootdir_hack$ac_datarootdir_seen" && + { ac_out=`sed -n '/\${datarootdir}/p' "$tmp/out"`; test -n "$ac_out"; } && + { ac_out=`sed -n '/^[ ]*datarootdir[ ]*:*=/p' "$tmp/out"`; test -z "$ac_out"; } && + { $as_echo "$as_me:$LINENO: WARNING: $ac_file contains a reference to the variable \`datarootdir' +which seems to be undefined. Please make sure it is defined." >&5 +$as_echo "$as_me: WARNING: $ac_file contains a reference to the variable \`datarootdir' +which seems to be undefined. Please make sure it is defined." >&2;} + + rm -f "$tmp/stdin" + case $ac_file in + -) cat "$tmp/out" && rm -f "$tmp/out";; + *) rm -f "$ac_file" && mv "$tmp/out" "$ac_file";; + esac \ + || { { $as_echo "$as_me:$LINENO: error: could not create $ac_file" >&5 +$as_echo "$as_me: error: could not create $ac_file" >&2;} + { (exit 1); exit 1; }; } + ;; + :H) + # + # CONFIG_HEADER + # + if test x"$ac_file" != x-; then + { + $as_echo "/* $configure_input */" \ + && eval '$AWK -f "$tmp/defines.awk"' "$ac_file_inputs" + } >"$tmp/config.h" \ + || { { $as_echo "$as_me:$LINENO: error: could not create $ac_file" >&5 +$as_echo "$as_me: error: could not create $ac_file" >&2;} + { (exit 1); exit 1; }; } + if diff "$ac_file" "$tmp/config.h" >/dev/null 2>&1; then + { $as_echo "$as_me:$LINENO: $ac_file is unchanged" >&5 +$as_echo "$as_me: $ac_file is unchanged" >&6;} + else + rm -f "$ac_file" + mv "$tmp/config.h" "$ac_file" \ + || { { $as_echo "$as_me:$LINENO: error: could not create $ac_file" >&5 +$as_echo "$as_me: error: could not create $ac_file" >&2;} + { (exit 1); exit 1; }; } + fi + else + $as_echo "/* $configure_input */" \ + && eval '$AWK -f "$tmp/defines.awk"' "$ac_file_inputs" \ + || { { $as_echo "$as_me:$LINENO: error: could not create -" >&5 +$as_echo "$as_me: error: could not create -" >&2;} + { (exit 1); exit 1; }; } + fi +# Compute "$ac_file"'s index in $config_headers. +_am_stamp_count=1 +for _am_header in $config_headers :; do + case $_am_header in + "$ac_file" | "$ac_file":* ) + break ;; + * ) + _am_stamp_count=`expr $_am_stamp_count + 1` ;; + esac +done +echo "timestamp for "$ac_file"" >`$as_dirname -- "$ac_file" || +$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$ac_file" : 'X\(//\)[^/]' \| \ + X"$ac_file" : 'X\(//\)$' \| \ + X"$ac_file" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X"$ac_file" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'`/stamp-h$_am_stamp_count + ;; + + :C) { $as_echo "$as_me:$LINENO: executing $ac_file commands" >&5 +$as_echo "$as_me: executing $ac_file commands" >&6;} + ;; + esac + + + case $ac_file$ac_mode in + "depfiles":C) test x"$AMDEP_TRUE" != x"" || for mf in $CONFIG_FILES; do + # Strip MF so we end up with the name of the file. + mf=`echo "$mf" | sed -e 's/:.*$//'` + # Check whether this is an Automake generated Makefile or not. + # We used to match only the files named `Makefile.in', but + # some people rename them; so instead we look at the file content. + # Grep'ing the first line is not enough: some people post-process + # each Makefile.in and add a new line on top of each file to say so. + # So let's grep whole file. + if grep '^#.*generated by automake' $mf > /dev/null 2>&1; then + dirpart=`$as_dirname -- "$mf" || +$as_expr X"$mf" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$mf" : 'X\(//\)[^/]' \| \ + X"$mf" : 'X\(//\)$' \| \ + X"$mf" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X"$mf" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'` + else + continue + fi + # Extract the definition of DEPDIR, am__include, and am__quote + # from the Makefile without running `make'. + DEPDIR=`sed -n 's/^DEPDIR = //p' < "$mf"` + test -z "$DEPDIR" && continue + am__include=`sed -n 's/^am__include = //p' < "$mf"` + test -z "am__include" && continue + am__quote=`sed -n 's/^am__quote = //p' < "$mf"` + # When using ansi2knr, U may be empty or an underscore; expand it + U=`sed -n 's/^U = //p' < "$mf"` + # Find all dependency output files, they are included files with + # $(DEPDIR) in their names. We invoke sed twice because it is the + # simplest approach to changing $(DEPDIR) to its actual value in the + # expansion. + for file in `sed -n " + s/^$am__include $am__quote\(.*(DEPDIR).*\)$am__quote"'$/\1/p' <"$mf" | \ + sed -e 's/\$(DEPDIR)/'"$DEPDIR"'/g' -e 's/\$U/'"$U"'/g'`; do + # Make sure the directory exists. + test -f "$dirpart/$file" && continue + fdir=`$as_dirname -- "$file" || +$as_expr X"$file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$file" : 'X\(//\)[^/]' \| \ + X"$file" : 'X\(//\)$' \| \ + X"$file" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X"$file" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'` + { as_dir=$dirpart/$fdir + case $as_dir in #( + -*) as_dir=./$as_dir;; + esac + test -d "$as_dir" || { $as_mkdir_p && mkdir -p "$as_dir"; } || { + as_dirs= + while :; do + case $as_dir in #( + *\'*) as_qdir=`$as_echo "$as_dir" | sed "s/'/'\\\\\\\\''/g"`;; #'( + *) as_qdir=$as_dir;; + esac + as_dirs="'$as_qdir' $as_dirs" + as_dir=`$as_dirname -- "$as_dir" || +$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$as_dir" : 'X\(//\)[^/]' \| \ + X"$as_dir" : 'X\(//\)$' \| \ + X"$as_dir" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X"$as_dir" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'` + test -d "$as_dir" && break + done + test -z "$as_dirs" || eval "mkdir $as_dirs" + } || test -d "$as_dir" || { { $as_echo "$as_me:$LINENO: error: cannot create directory $as_dir" >&5 +$as_echo "$as_me: error: cannot create directory $as_dir" >&2;} + { (exit 1); exit 1; }; }; } + # echo "creating $dirpart/$file" + echo '# dummy' > "$dirpart/$file" + done +done + ;; + + esac +done # for ac_tag + + +{ (exit 0); exit 0; } +_ACEOF +chmod +x $CONFIG_STATUS +ac_clean_files=$ac_clean_files_save + +test $ac_write_fail = 0 || + { { $as_echo "$as_me:$LINENO: error: write failure creating $CONFIG_STATUS" >&5 +$as_echo "$as_me: error: write failure creating $CONFIG_STATUS" >&2;} + { (exit 1); exit 1; }; } + + +# configure is writing to config.log, and then calls config.status. +# config.status does its own redirection, appending to config.log. +# Unfortunately, on DOS this fails, as config.log is still kept open +# by configure, so config.status won't be able to write to it; its +# output is simply discarded. So we exec the FD to /dev/null, +# effectively closing config.log, so it can be properly (re)opened and +# appended to by config.status. When coming back to configure, we +# need to make the FD available again. +if test "$no_create" != yes; then + ac_cs_success=: + ac_config_status_args= + test "$silent" = yes && + ac_config_status_args="$ac_config_status_args --quiet" + exec 5>/dev/null + $SHELL $CONFIG_STATUS $ac_config_status_args || ac_cs_success=false + exec 5>>config.log + # Use ||, not &&, to avoid exiting from the if with $? = 1, which + # would make configure fail if this is the last instruction. + $ac_cs_success || { (exit 1); exit 1; } +fi +if test -n "$ac_unrecognized_opts" && test "$enable_option_checking" != no; then + { $as_echo "$as_me:$LINENO: WARNING: unrecognized options: $ac_unrecognized_opts" >&5 +$as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2;} +fi + diff --git a/mosesdecoder/contrib/memscore/m4/ax_boost_base.m4 b/mosesdecoder/contrib/memscore/m4/ax_boost_base.m4 new file mode 100644 index 0000000000000000000000000000000000000000..2e5afd09158fbb604354a37441f18cf7458996c2 --- /dev/null +++ b/mosesdecoder/contrib/memscore/m4/ax_boost_base.m4 @@ -0,0 +1,223 @@ +# =========================================================================== +# http://autoconf-archive.cryp.to/ax_boost_base.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_BOOST_BASE([MINIMUM-VERSION]) +# +# DESCRIPTION +# +# Test for the Boost C++ libraries of a particular version (or newer) +# +# If no path to the installed boost library is given the macro searchs +# under /usr, /usr/local, /opt and /opt/local and evaluates the +# $BOOST_ROOT environment variable. Further documentation is available at +# . +# +# This macro calls: +# +# AC_SUBST(BOOST_CPPFLAGS) / AC_SUBST(BOOST_LDFLAGS) +# +# And sets: +# +# HAVE_BOOST +# +# LAST MODIFICATION +# +# 2008-04-12 +# +# COPYLEFT +# +# Copyright (c) 2008 Thomas Porschberg +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. + +AC_DEFUN([AX_BOOST_BASE], +[ +AC_ARG_WITH([boost], + AS_HELP_STRING([--with-boost@<:@=DIR@:>@], [use boost (default is yes) - it is possible to specify the root directory for boost (optional)]), + [ + if test "$withval" = "no"; then + want_boost="no" + elif test "$withval" = "yes"; then + want_boost="yes" + ac_boost_path="" + else + want_boost="yes" + ac_boost_path="$withval" + fi + ], + [want_boost="yes"]) + + +AC_ARG_WITH([boost-libdir], + AS_HELP_STRING([--with-boost-libdir=LIB_DIR], + [Force given directory for boost libraries. Note that this will overwrite library path detection, so use this parameter only if default library detection fails and you know exactly where your boost libraries are located.]), + [ + if test -d $withval + then + ac_boost_lib_path="$withval" + else + AC_MSG_ERROR(--with-boost-libdir expected directory name) + fi + ], + [ac_boost_lib_path=""] +) + +if test "x$want_boost" = "xyes"; then + boost_lib_version_req=ifelse([$1], ,1.20.0,$1) + boost_lib_version_req_shorten=`expr $boost_lib_version_req : '\([[0-9]]*\.[[0-9]]*\)'` + boost_lib_version_req_major=`expr $boost_lib_version_req : '\([[0-9]]*\)'` + boost_lib_version_req_minor=`expr $boost_lib_version_req : '[[0-9]]*\.\([[0-9]]*\)'` + boost_lib_version_req_sub_minor=`expr $boost_lib_version_req : '[[0-9]]*\.[[0-9]]*\.\([[0-9]]*\)'` + if test "x$boost_lib_version_req_sub_minor" = "x" ; then + boost_lib_version_req_sub_minor="0" + fi + WANT_BOOST_VERSION=`expr $boost_lib_version_req_major \* 100000 \+ $boost_lib_version_req_minor \* 100 \+ $boost_lib_version_req_sub_minor` + AC_MSG_CHECKING(for boostlib >= $boost_lib_version_req) + succeeded=no + + dnl first we check the system location for boost libraries + dnl this location ist chosen if boost libraries are installed with the --layout=system option + dnl or if you install boost with RPM + if test "$ac_boost_path" != ""; then + BOOST_LDFLAGS="-L$ac_boost_path/lib" + BOOST_CPPFLAGS="-I$ac_boost_path/include" + else + for ac_boost_path_tmp in /usr /usr/local /opt /opt/local ; do + if test -d "$ac_boost_path_tmp/include/boost" && test -r "$ac_boost_path_tmp/include/boost"; then + BOOST_LDFLAGS="-L$ac_boost_path_tmp/lib" + BOOST_CPPFLAGS="-I$ac_boost_path_tmp/include" + break; + fi + done + fi + + dnl overwrite ld flags if we have required special directory with + dnl --with-boost-libdir parameter + if test "$ac_boost_lib_path" != ""; then + BOOST_LDFLAGS="-L$ac_boost_lib_path" + fi + + CPPFLAGS_SAVED="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + export CPPFLAGS + + LDFLAGS_SAVED="$LDFLAGS" + LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" + export LDFLAGS + + AC_LANG_PUSH(C++) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + @%:@include + ]], [[ + #if BOOST_VERSION >= $WANT_BOOST_VERSION + // Everything is okay + #else + # error Boost version is too old + #endif + ]])],[ + AC_MSG_RESULT(yes) + succeeded=yes + found_system=yes + ],[ + ]) + AC_LANG_POP([C++]) + + + + dnl if we found no boost with system layout we search for boost libraries + dnl built and installed without the --layout=system option or for a staged(not installed) version + if test "x$succeeded" != "xyes"; then + _version=0 + if test "$ac_boost_path" != ""; then + if test -d "$ac_boost_path" && test -r "$ac_boost_path"; then + for i in `ls -d $ac_boost_path/include/boost-* 2>/dev/null`; do + _version_tmp=`echo $i | sed "s#$ac_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'` + V_CHECK=`expr $_version_tmp \> $_version` + if test "$V_CHECK" = "1" ; then + _version=$_version_tmp + fi + VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'` + BOOST_CPPFLAGS="-I$ac_boost_path/include/boost-$VERSION_UNDERSCORE" + done + fi + else + for ac_boost_path in /usr /usr/local /opt /opt/local ; do + if test -d "$ac_boost_path" && test -r "$ac_boost_path"; then + for i in `ls -d $ac_boost_path/include/boost-* 2>/dev/null`; do + _version_tmp=`echo $i | sed "s#$ac_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'` + V_CHECK=`expr $_version_tmp \> $_version` + if test "$V_CHECK" = "1" ; then + _version=$_version_tmp + best_path=$ac_boost_path + fi + done + fi + done + + VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'` + BOOST_CPPFLAGS="-I$best_path/include/boost-$VERSION_UNDERSCORE" + if test "$ac_boost_lib_path" = "" + then + BOOST_LDFLAGS="-L$best_path/lib" + fi + + if test "x$BOOST_ROOT" != "x"; then + if test -d "$BOOST_ROOT" && test -r "$BOOST_ROOT" && test -d "$BOOST_ROOT/stage/lib" && test -r "$BOOST_ROOT/stage/lib"; then + version_dir=`expr //$BOOST_ROOT : '.*/\(.*\)'` + stage_version=`echo $version_dir | sed 's/boost_//' | sed 's/_/./g'` + stage_version_shorten=`expr $stage_version : '\([[0-9]]*\.[[0-9]]*\)'` + V_CHECK=`expr $stage_version_shorten \>\= $_version` + if test "$V_CHECK" = "1" -a "$ac_boost_lib_path" = "" ; then + AC_MSG_NOTICE(We will use a staged boost library from $BOOST_ROOT) + BOOST_CPPFLAGS="-I$BOOST_ROOT" + BOOST_LDFLAGS="-L$BOOST_ROOT/stage/lib" + fi + fi + fi + fi + + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + export CPPFLAGS + LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" + export LDFLAGS + + AC_LANG_PUSH(C++) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + @%:@include + ]], [[ + #if BOOST_VERSION >= $WANT_BOOST_VERSION + // Everything is okay + #else + # error Boost version is too old + #endif + ]])],[ + AC_MSG_RESULT(yes) + succeeded=yes + found_system=yes + ],[ + ]) + AC_LANG_POP([C++]) + fi + + if test "$succeeded" != "yes" ; then + if test "$_version" = "0" ; then + AC_MSG_ERROR([[We could not detect the boost libraries (version $boost_lib_version_req_shorten or higher). If you have a staged boost library (still not installed) please specify \$BOOST_ROOT in your environment and do not give a PATH to --with-boost option. If you are sure you have boost installed, then check your version number looking in . See http://randspringer.de/boost for more documentation.]]) + else + AC_MSG_NOTICE([Your boost libraries seems to old (version $_version).]) + fi + else + AC_SUBST(BOOST_CPPFLAGS) + AC_SUBST(BOOST_LDFLAGS) + AC_DEFINE(HAVE_BOOST,,[define if the Boost library is available]) + fi + + CPPFLAGS="$CPPFLAGS_SAVED" + LDFLAGS="$LDFLAGS_SAVED" +fi + +]) diff --git a/mosesdecoder/contrib/memscore/scorer.cpp b/mosesdecoder/contrib/memscore/scorer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e8cf4ce494fd9eceb90078fe17d1ee89ce896d60 --- /dev/null +++ b/mosesdecoder/contrib/memscore/scorer.cpp @@ -0,0 +1,402 @@ +// memscore - in-memory phrase scoring for Statistical Machine Translation +// Christian Hardmeier, FBK-irst, Trento, 2010 +// $Id$ + +#include +#include +#include + +#include "scorer-impl.h" +#include "timestamp.h" +#include "lexdecom.h" + +#ifdef ENABLE_CHANNEL_SCORER +#include "channel-scorer.h" +#endif + +const std::vector &PhraseScorerFactory::scorer_list() +{ + static std::vector list; + if(list.size() == 0) { + list.push_back("ml - maximum likelihood score (relative frequency)"); + list.push_back("wittenbell - Witten-Bell smoothing"); + list.push_back("absdiscount - absolute discounting"); + list.push_back("kndiscount1 - Knesser-Ney discounting"); + list.push_back("kndiscount3 - modified Knesser-Ney discounting"); + list.push_back("lexweights - lexical weights (Koehn et al., NAACL 2003)"); +#ifdef ENABLE_CHANNEL_SCORER + list.push_back("channel - channel adaptation"); +#endif + list.push_back("const - constant phrase penalty"); + list.push_back("lexdecomp - lexical decomposition smoothing"); + } + + return list; +} + +PhraseScorer *PhraseScorerFactory::create_scorer(const char *argv[], int &argp, bool reverse) +{ + const char *arg = argv[argp++]; + if(arg == NULL) + usage(); + + if(!strcmp(arg, "ml")) + return MLPhraseScorer::create_scorer(argv, argp, reverse, *this); + else if(!strcmp(arg, "wittenbell")) + return WittenBellPhraseScorer::create_scorer(argv, argp, reverse, *this); + else if(!strcmp(arg, "absdiscount")) + return AbsoluteDiscountPhraseScorer::create_scorer(argv, argp, reverse, *this); + else if(!strcmp(arg, "kndiscount1")) + return KNDiscount1PhraseScorer::create_scorer(argv, argp, reverse, *this); + else if(!strcmp(arg, "kndiscount3")) + return KNDiscount3PhraseScorer::create_scorer(argv, argp, reverse, *this); + else if(!strcmp(arg, "lexweights")) + return LexicalWeightPhraseScorer::create_scorer(argv, argp, reverse, *this); +#ifdef ENABLE_CHANNEL_SCORER + else if(!strcmp(arg, "channel")) + return ChannelAdaptationPhraseScorer::create_scorer(argv, argp, reverse, *this); +#endif + else if(!strcmp(arg, "const")) + return ConstantPhraseScorer::create_scorer(argv, argp, reverse, *this); + else if (!strcmp(arg, "lexdecomp")) + return LexicalDecompositionPhraseScorer::create_scorer(argv, argp, reverse, *this); + else { + std::cerr << "Unknown phrase scorer type: " << arg << std::endl << std::endl; + usage(); + } +} + +PhraseScorer *MLPhraseScorer::create_scorer(const char *argv[], int &argp, bool reverse, const PhraseScorerFactory &ptf) +{ + return new MLPhraseScorer(ptf.get_phrase_table(), reverse); +} + +#if 1 +void MLPhraseScorer::do_score_phrases() {} +#else +void MLPhraseScorer::do_score_phrases() +{ + Score bla = 0; + Timestamp t_it; + for(Count i = 0; i < 200; i++) { + for(PhraseTable::iterator it = phrase_table_.begin(); it != phrase_table_.end(); ++it) { + PhrasePairInfo ppair = *it; + Phrase tgt = ppair.get_tgt(); + bla += bla * ppair.get_count() / phrase_table_.get_tgt_phrase(tgt).get_count(); + } + } + std::cerr << "Time for 200 iterations with ML estimation: " << (t_it.elapsed_time() / 1000) << " ms" << std::endl; + std::cerr << bla << std::endl; +} +#endif + +Score MLPhraseScorer::do_get_score(const PhraseTable::const_iterator &it) +{ + PhraseInfo &tgt_phrase = phrase_table_.get_tgt_phrase(it->get_tgt()); + return static_cast(it->get_count()) / tgt_phrase.get_count(); +} + +PhraseScorer *WittenBellPhraseScorer::create_scorer(const char *argv[], int &argp, bool reverse, const PhraseScorerFactory &ptf) +{ + return new WittenBellPhraseScorer(ptf.get_phrase_table(), reverse); +} + +Score WittenBellPhraseScorer::do_get_score(const PhraseTable::const_iterator &it) +{ + PhraseInfo &tgt_phrase = phrase_table_.get_tgt_phrase(it->get_tgt()); + return static_cast(it->get_count()) / (tgt_phrase.get_count() + tgt_phrase.get_distinct()); +} + + +PhraseScorer *AbsoluteDiscountPhraseScorer::create_scorer(const char *argv[], int &argp, bool reverse, const PhraseScorerFactory &ptf) +{ + return new AbsoluteDiscountPhraseScorer(ptf.get_phrase_table(), reverse); +} +// p(s|t) = (c(s,t) - beta) / c(t) <-- absolute discounting + +void AbsoluteDiscountPhraseScorer::do_score_phrases() +{ + Count n1 = 0, n2 = 0; + + for(PhraseTable::iterator it = phrase_table_.begin(); it != phrase_table_.end(); ++it) { + PhrasePairInfo ppinfo = *it; + Count c = ppinfo.get_count(); + switch(c) { + case 1: + n1++; + break; + case 2: + n2++; + } + } + + discount_ = static_cast(n1) / (n1 + 2*n2); +} + +inline Score AbsoluteDiscountPhraseScorer::get_discount() +{ + return discount_; +} + +Score AbsoluteDiscountPhraseScorer::do_get_score(const PhraseTable::const_iterator &it) +{ + PhraseInfo &tgt_phrase = phrase_table_.get_tgt_phrase(it->get_tgt()); + return (it->get_count() - discount_) / tgt_phrase.get_count(); +} + +PhraseScorer *KNDiscount1PhraseScorer::create_scorer(const char *argv[], int &argp, bool reverse, const PhraseScorerFactory &ptf) +{ + return new KNDiscount1PhraseScorer(ptf.get_phrase_table(), reverse); +} + + +void KNDiscount1PhraseScorer::do_score_phrases() +{ + Count n1 = 0, n2 = 0; + Count total_count = 0; + + for(PhraseTable::iterator it = phrase_table_.begin(); it != phrase_table_.end(); ++it) { + PhrasePairInfo ppinfo = *it; + PhraseInfo &tgt_phrase = phrase_table_.get_tgt_phrase(it->get_tgt()); + total_count += tgt_phrase.get_count(); + + Count c = ppinfo.get_count(); + switch(c) { + case 1: + n1++; + break; + case 2: + n2++; + } + } + + discount_ = static_cast(n1) / (n1 + 2*n2); + total_count_ = static_cast(total_count); + +} + +Score KNDiscount1PhraseScorer::do_get_score(const PhraseTable::const_iterator &it) +{ + PhraseInfo &tgt_phrase = phrase_table_.get_tgt_phrase(it->get_tgt()); + PhraseInfo &src_phrase = phrase_table_.get_src_phrase(it->get_src()); + return ((it->get_count() - discount_) / tgt_phrase.get_count()) + (discount_ * tgt_phrase.get_distinct() / tgt_phrase.get_count())*(src_phrase.get_count() / total_count_); +} + +PhraseScorer *KNDiscount3PhraseScorer::create_scorer(const char *argv[], int &argp, bool reverse, const PhraseScorerFactory &ptf) +{ + return new KNDiscount3PhraseScorer(ptf.get_phrase_table(), reverse); +} + + +void KNDiscount3PhraseScorer::do_score_phrases() +{ + Count n1 = 0, n2 = 0, n3 = 0, n4 = 0; + Count total_count = 0; //total number of source or target phrases (including repetitions) + Count total_distinct_n1 = 0; //sum_{s} n1plus(s,*) + Count total_distinct_n2 = 0; + Count total_distinct_n3plus = 0; + Score y; + + for(PhraseTable::iterator it = phrase_table_.begin(); it != phrase_table_.end(); ++it) { + PhrasePairInfo ppinfo = *it; + PhraseInfo &src_phrase = phrase_table_.get_src_phrase(it->get_src()); + PhraseInfo &tgt_phrase = phrase_table_.get_tgt_phrase(it->get_tgt()); + + total_count += src_phrase.get_count(); + + Count c = ppinfo.get_count(); + switch(c) { + case 1: + n1++; + tgt_phrase.inc_n1(); + src_phrase.inc_n1(); + total_distinct_n1++; + break; + case 2: + n2++; + tgt_phrase.inc_n2(); + src_phrase.inc_n2(); + total_distinct_n2++; + break; + case 3: + n3++; + tgt_phrase.inc_n3plus(); + src_phrase.inc_n3plus(); + total_distinct_n3plus++; + break; + case 4: + n4++; + tgt_phrase.inc_n3plus(); + src_phrase.inc_n3plus(); + total_distinct_n3plus++; + } + + } + + y = (Score)(n1) / (n1 + 2*n2); + discount1_ = static_cast (1) - (2)*(y)*(n2 / n1); + discount2_ = static_cast (2) - (3)*(y)*(n3 / n2); + discount3plus_ = static_cast (3) - (4)*(y)*(n4 / n3); + total_distinct_n1_ = static_cast(total_distinct_n1); + total_distinct_n2_ = static_cast(total_distinct_n2); + total_distinct_n3plus_ = static_cast(total_distinct_n3plus); +} + +Score KNDiscount3PhraseScorer::do_get_score(const PhraseTable::const_iterator &it) +{ + PhrasePairInfo ppinfo = *it; + PhraseInfo &tgt_phrase = phrase_table_.get_tgt_phrase(it->get_tgt()); + PhraseInfo &src_phrase = phrase_table_.get_src_phrase(it->get_src()); + + Score norm = (discount1_ * tgt_phrase.get_n1() + discount2_ * tgt_phrase.get_n2() + discount3plus_ * tgt_phrase.get_n3plus()) / tgt_phrase.get_count(); + Count c = ppinfo.get_count(); + switch(c) { + case 1: + return ((it->get_count() - discount1_) / tgt_phrase.get_count()) + \ + norm*(src_phrase.get_n1() / total_distinct_n1_); + break; + case 2: + return ((it->get_count() - discount2_) / tgt_phrase.get_count()) + \ + norm*(src_phrase.get_n2() / total_distinct_n2_); + break; + default: + return ((it->get_count() - discount3plus_) / tgt_phrase.get_count()) + \ + norm*(src_phrase.get_n3plus() / total_distinct_n3plus_); + } +} + +PhraseScorer *LexicalWeightPhraseScorer::create_scorer(const char *argv[], int &argp, bool reverse, const PhraseScorerFactory &ptf) +{ + bool overall_max = true; + + if(argv[argp] == NULL) + usage(); + + if(!strcmp(argv[argp], "-AlignmentCount")) { + overall_max = false; + argp++; + if(argv[argp] == NULL) + usage(); + } + + String lwfile(argv[argp++]); + return new LexicalWeightPhraseScorer(ptf.get_phrase_table(), reverse, lwfile, overall_max); +} + +LexicalWeightPhraseScorer::LexicalWeightPhraseScorer(PhraseTable &pd, bool reverse, const String &weightfile, bool overall_max) : + PhraseScorer(pd, reverse), overall_max_score_(overall_max), null_word_(PhraseText::index_word("NULL")) +{ + std::ifstream wfile(weightfile.c_str()); + + while(!wfile.eof()) { + if(wfile.fail()) { + std::cerr << "Problem reading file: " << weightfile << std::endl; + exit(1); + } + + String src, tgt; + Score weight; + + wfile >> src >> tgt >> weight; + Count src_id = PhraseText::index_word(src); + Count tgt_id = PhraseText::index_word(tgt); + weight_map_.insert(std::make_pair(std::make_pair(src_id, tgt_id), weight)); + } + + wfile.close(); +} + +Score LexicalWeightPhraseScorer::get_weight(const String &s_src, const String &s_tgt) const +{ + Count src = PhraseText::index_word(s_src); + Count tgt = PhraseText::index_word(s_tgt); + return get_weight(src, tgt); +} + +inline Score LexicalWeightPhraseScorer::get_weight(Count src, Count tgt) const +{ + WeightMapType_::const_iterator it = weight_map_.find(std::make_pair(src, tgt)); + if(it == weight_map_.end()) + return 0.00001; // default value copied from Philipp Koehn's scorer + return it->second; +} + +#if 1 +void LexicalWeightPhraseScorer::do_score_phrases() {} +#else +void LexicalWeightPhraseScorer::do_score_phrases() +{ + Score bla = 0; + Timestamp t_it; + for(Count i = 0; i < 200; i++) { + for(PhraseTable::iterator it = phrase_table_.begin(); it != phrase_table_.end(); ++it) { + PhrasePairInfo ppair = *it; + Phrase src = ppair.get_src(); + Phrase tgt = ppair.get_tgt(); + bla += bla * get_score(src, tgt); + } + } + std::cerr << "Time for 200 iterations with lexical weights: " << (t_it.elapsed_time() / 1000) << " ms" << std::endl; + std::cerr << bla << std::endl; +} +#endif + +Score LexicalWeightPhraseScorer::do_get_score(const PhraseTable::const_iterator &it) +{ + const Phrase src = it->get_src(); + const Phrase tgt = it->get_tgt(); + const PhraseText &src_phrase = phrase_table_.get_src_phrase(src).get_phrase(); + const PhraseText &tgt_phrase = phrase_table_.get_tgt_phrase(tgt).get_phrase(); + const PhrasePairInfo &ppair = *it; + + Count max_count = 0; + + Score maxlex = 0; + PhrasePairInfo::AlignmentVector av = ppair.get_alignments(); + for(PhrasePairInfo::AlignmentVector::const_iterator it = av.begin(); it != av.end(); ++it) { + const PhraseAlignment &alig = it->first; + const Count alig_cnt = it->second; + + assert(alig.get_source_length() == src_phrase.size() && alig.get_target_length() == tgt_phrase.size()); + + if(!overall_max_score_ && alig_cnt < max_count) + continue; + max_count = alig_cnt; + + Score lex = 1; + for(Count s = 0; s < src_phrase.size(); s++) { + Score factor = 0; + Count na = 0; + for(Count t = 0; t < tgt_phrase.size(); t++) + if(alig.is_aligned(s, t)) { + const Score w = get_weight(src_phrase[s], tgt_phrase[t]); + factor += w; + na++; + } + + if(na > 0) + lex *= factor / na; + else + lex *= get_weight(src_phrase[s], null_word_); + } + + if(lex > maxlex) + maxlex = lex; + } + + return maxlex; +} + +PhraseScorer *ConstantPhraseScorer::create_scorer(const char *argv[], int &argp, bool reverse, const PhraseScorerFactory &ptf) +{ + if(argv[argp] == NULL) + usage(); + Score c = atof(argv[argp++]); + return new ConstantPhraseScorer(ptf.get_phrase_table(), reverse, c); +} + +Score ConstantPhraseScorer::do_get_score(const PhraseTable::const_iterator &it) +{ + return constant_; +} + diff --git a/mosesdecoder/contrib/memscore/statistic.h b/mosesdecoder/contrib/memscore/statistic.h new file mode 100644 index 0000000000000000000000000000000000000000..2061a6608ed676003a4b18a9c8065b1d0f302f48 --- /dev/null +++ b/mosesdecoder/contrib/memscore/statistic.h @@ -0,0 +1,26 @@ +// memscore - in-memory phrase scoring for Statistical Machine Translation +// Christian Hardmeier, FBK-irst, Trento, 2010 +// $Id$ + +#ifndef STATISTIC_H +#define STATISTIC_H + +#include "memscore.h" + +class PhraseInfoList; + +class PhraseStatistic +{ +protected: + bool computation_done_; + +public: + PhraseStatistic() : computation_done_(false) {} + virtual ~PhraseStatistic() {} + + virtual void attach(PhraseInfoList &pilist) = 0; + virtual void compute_statistic() = 0; + virtual Score get_score(PhraseInfo &pi) = 0; +}; + +#endif diff --git a/mosesdecoder/contrib/mert-sge-nosync/README b/mosesdecoder/contrib/mert-sge-nosync/README new file mode 100644 index 0000000000000000000000000000000000000000..85b95ec311a4ee0b91239e3f128ca15f0544152f --- /dev/null +++ b/mosesdecoder/contrib/mert-sge-nosync/README @@ -0,0 +1,29 @@ +MERT-sge-nosync +Raymond Ng, University of Sheffield. +Apr, 2014. + +The parallel MERT tuning scripts in moses cannot run in the SGE "no-sync" mode (where job submission is done in one go, after which user can go offline while leaving the SGE to manage the whole process). Scripts provided in this site are for parallel MERT in SGE no-sync mode. You will need to have SSH support in perl (Step 2), and set up public ssh-keys between the running hosts and the submission hosts (Step 3). + +1. Untar the scipts in ${MOSES}/scripts/ + +2. Download and install OpenSSH packages for perl: +Net-OpenSSH-Compat (http://search.cpan.org/CPAN/authors/id/S/SA/SALVA/Net-OpenSSH-Compat-0.06.tar.gz) +$ cd Net-OpenSSH-Compat-0.06 +$ perl Makefile.PL +(You may have dependency issue and need to install IO-Tty and Net-OpenSSH in advance) +IO-Tty-1.10 (http://search.cpan.org/CPAN/authors/id/T/TO/TODDR/IO-Tty-1.10.tar.gz) +Net-OpenSSH-0.60 (http://search.cpan.org/CPAN/authors/id/S/SA/SALVA/Net-OpenSSH-0.60.tar.gz) + +3. Set up public ssh-keys for accessing the submithost (machine from which qsub is executed) from the running hosts (machines which actually runs the scripts) +http://www.linuxproblem.org/art_9.html + +4. Run parallel MERT by +nohup nice ${MOSES}/scripts/training/mert-moses-sge-nosync.pl ${TM_DEVTEXT_SOURCE} \ +--threads 20 --jobs 20 \ +--queue-flags='-q normal.q -P project' \ +--submithost='squeal' \ +${TM_DEVTEXT_TARGET} ${MOSES}/dist/bin/moses ${WORKINGDIR}/train/model/moses.ini \ +--mertdir ${MOSES}/dist/bin/ >& ${WORKINGDIR}/mert.out + + +(tested on moses version Built Apr 2012 version) diff --git a/mosesdecoder/contrib/mert-sge-nosync/generic/moses-parallel-sge-nosync.pl b/mosesdecoder/contrib/mert-sge-nosync/generic/moses-parallel-sge-nosync.pl new file mode 100644 index 0000000000000000000000000000000000000000..dc9d695713dfb7c61d1967c5daffe26c697ead47 --- /dev/null +++ b/mosesdecoder/contrib/mert-sge-nosync/generic/moses-parallel-sge-nosync.pl @@ -0,0 +1,1388 @@ +#! /usr/bin/perl + +# $Id$ +####################### +# Revision history +# +# 02 Aug 2006 added strict requirement +# 01 Aug 2006 fix bug about inputfile parameter +# fix bug about suffix index generation +# 31 Jul 2006 added parameter for reading queue parameters +# 29 Jul 2006 added code to handling consfusion networks +# 28 Jul 2006 added a better policy for removing jobs from the queue in case of killing signal (CTRL-C) +# added the parameter -qsub-prefix which sets the prefix for the name of submitted jobs +# 27 Jul 2006 added safesystem() function and other checks to handle with process failure +# added checks for existence of decoder and configuration file +# 26 Jul 2006 fix a bug related to the use of absolute path for srcfile and nbestfile + +use strict; +use Net::OpenSSH::Compat::Perl; + + +####################### +#Customizable parameters + +#parameters for submiiting processes through Sun GridEngine +my $submithost = undef; +my $queueparameters=""; +my $batch_and_join = undef; +my $processid="$$"; + +# look for the correct pwdcmd +my $pwdcmd = getPwdCmd(); + +my $workingdir = `$pwdcmd`; chomp $workingdir; +my $tmpdir="$workingdir/tmp$$"; +my $splitpfx="split$$"; + + +$SIG{'INT'} = \&kill_all_and_quit; # catch exception for CTRL-C + +####################### +#Default parameters +my $jobscript="$workingdir/job$$"; +my $qsubout="$workingdir/out.job$$"; +my $qsuberr="$workingdir/err.job$$"; + +my $mosesparameters=""; +my $feed_moses_via_stdin = 0; + # a workaround, for a reason, the default "-input-file X" blocks + # my moses, while "< X" works fine. +my $cfgfile=""; #configuration file + +my $version=undef; +my $help=0; +my $dbg=0; +my $jobs=4; +my $mosescmd="$ENV{MOSESBIN}/moses"; #decoder in use +my $inputlist=undef; +my $inputfile=undef; +my $inputtype=0; +my @nbestlist=(); +my $nbestlist=undef; +my $nbestfile=undef; +my $oldnbestfile=undef; +my $oldnbest=undef; +my $nbestflag=0; +my @wordgraphlist=(); +my $wordgraphlist=undef; +my $wordgraphfile=undef; +my $wordgraphflag=0; +my $robust=5; # resubmit crashed jobs robust-time +my $alifile=undef; +my $outnbest=undef; +my $logfile=""; +my $logflag=""; +my $searchgraphlist=""; +my $searchgraphfile=""; +my $searchgraphflag=0; +my $qsubname="MOSES"; +my $qsubwrapper=undef; +my $qsubwrapper_exit=undef; +my $old_sge = 0; # assume old Sun Grid Engine (<6.0) where qsub does not + # implement -sync and -b +my $___LATTICE_SAMPLES = 0; +my $___DECODER_FLAGS = ""; # additional parameters to pass to the decoder +my $___N_BEST_LIST_SIZE = 100; +my $___RANGES = undef; +my $___WORKING_DIR = undef; +my $SCRIPTS_ROOTDIR = undef; +my $postdecodecmd = undef; +my $postdecodeargs = undef; + + +my $run = 0; +my $jobid = -1; +my $prevjid = undef; +my $need_to_normalize = 1; + +####################### +# Command line options processing +sub init(){ + + + + use Getopt::Long qw(:config pass_through no_ignore_case permute); + GetOptions('version'=>\$version, + 'help'=>\$help, + 'debug'=>\$dbg, + 'jobs=i'=>\$jobs, + 'decoder=s'=> \$mosescmd, + 'robust=i' => \$robust, + 'script-rootdir=s' => \$SCRIPTS_ROOTDIR, + 'lattice-samples=i' => \$___LATTICE_SAMPLES, + 'decoder-flags=s' => \$___DECODER_FLAGS, + 'feed-decoder-via-stdin'=> \$feed_moses_via_stdin, + 'logfile=s'=> \$logfile, + 'i|inputfile|input-file=s'=> \$inputlist, + 'n-best-list-size=s'=> \$___N_BEST_LIST_SIZE, + 'n-best-file=s'=> \$oldnbestfile, + 'n-best-size=i'=> \$oldnbest, + 'output-search-graph|osg=s'=> \$searchgraphlist, + 'output-word-graph|owg=s'=> \$wordgraphlist, + 'alignment-output-file=s'=> \$alifile, + 'submithost=s'=> \$submithost, + 'queue-parameters=s'=> \$queueparameters, + 'inputtype=i'=> \$inputtype, + 'config|f=s'=>\$cfgfile, + 'ranges=s@'=> \$___RANGES, + 'old-sge' => \$old_sge, + 'run=i' => \$run, + 'need-to-normalize' => \$need_to_normalize, + 'working-dir=s' => \$___WORKING_DIR, + 'qsubwrapper=s' => \$qsubwrapper, + 'qsubwrapper-exit=s' => \$qsubwrapper_exit + ) or exit(1); + +# 'decoder-parameters=s'=> \$mosesparameters, +# 'n-best-list=s'=> \$nbestlist, +# 'qsub-prefix=s'=> \$qsubname, +} + +sub init_secondpart() { + getNbestParameters(); + + getSearchGraphParameters(); + + getWordGraphParameters(); + + getLogParameters(); + + chomp(my $my_username = `whoami`); + # $submithost = "squeal"; + + print STDERR "submithost is $submithost\n"; + + my $prevjid = undef; + my $jobid = undef; + +#print_parameters(); +#print STDERR "nbestflag:$nbestflag\n"; +#print STDERR "searchgraphflag:$searchgraphflag\n"; +print STDERR "wordgraphflag:$wordgraphflag\n"; +#print STDERR "inputlist:$inputlist\n"; + + chomp($inputfile=`basename $inputlist`) if defined($inputlist); + + # $mosesparameters.="@ARGV -config $cfgfile -inputtype $inputtype"; + # $mosesparameters = "@ARGV -config $cfgfile -inputtype $inputtype "; + # $mosesparameters .= "@ARGV -config $cfgfile -inputtype $inputtype "; + $mosesparameters .= " -config $cfgfile -inputtype $inputtype "; + +} + + +####################### +##print version +sub version(){ +# print STDERR "version 1.0 (15-07-2006)\n"; +# print STDERR "version 1.1 (17-07-2006)\n"; +# print STDERR "version 1.2 (18-07-2006)\n"; +# print STDERR "version 1.3 (21-07-2006)\n"; +# print STDERR "version 1.4 (26-07-2006)\n"; +# print STDERR "version 1.5 (27-07-2006)\n"; +# print STDERR "version 1.6 (28-07-2006)\n"; +# print STDERR "version 1.7 (29-07-2006)\n"; +# print STDERR "version 1.8 (31-07-2006)\n"; +# print STDERR "version 1.9 (01-08-2006)\n"; +# print STDERR "version 1.10 (02-08-2006)\n"; +# print STDERR "version 1.11 (10-10-2006)\n"; +# print STDERR "version 1.12 (27-12-2006)\n"; +# print STDERR "version 1.13 (29-12-2006)\n"; + print STDERR "version 1.13b (01-04-2014)\n"; + exit(1); +} + +#usage +sub usage(){ + print STDERR "moses-parallel.pl [parallel-options] [moses-options]\n"; + print STDERR "Options marked (*) are required.\n"; + print STDERR "Parallel options:\n"; + print STDERR "* -decoder Moses decoder to use\n"; + print STDERR "* -i|inputfile|input-file the input text to translate\n"; + print STDERR "* -jobs number of required jobs\n"; + print STDERR " -logfile file where storing log files of all jobs\n"; + print STDERR " -queue-parameters specific requirements for queue\n"; + print STDERR " -old-sge Assume Sun Grid Engine < 6.0\n"; + print STDERR " -debug debug\n"; + print STDERR " -version print version of the script\n"; + print STDERR " -help this help\n"; + print STDERR "Moses options:\n"; + print STDERR " -inputtype <0|1|2> 0 for text, 1 for confusion networks, 2 for lattices\n"; + print STDERR " -output-search-graph (osg) : Output connected hypotheses of search into specified filename\n"; + print STDERR " -output-word-graph (osg) ' <0|1>': Output stack info as word graph. Takes filename, 0=only hypos in stack, 1=stack + nbest hypos\n"; + print STDERR " IMPORTANT NOTE: use single quote to group parameters of -output-word-graph\n"; + print STDERR " This is different from standard moses\n"; + print STDERR " -lattice-samples : how many lattice samples (Chatterjee & Cancedda, emnlp 2010) (added option in moses-parallel-sge-nosync)\n"; + print STDERR " -n-best-list-size : size of nbest lists (added option in moses-parallel-sge-nosync)\n"; + print STDERR " NOTE: -n-best-file-n-best-size are passed to the decoder as \"-n-best-list \"\n"; + print STDERR " -decoder-flags : (added option is moses-parallel-sge-nosync)\n"; + print STDERR " -ranges : (added option is moses-parallel-sge-nosync)\n"; + print STDERR " -run : (add option in moses-parallel-sge-nosync)\n"; + print STDERR "* -config (f) configuration file\n"; + print STDERR "All other options are passed to Moses\n"; + print STDERR " (This way to pass parameters is maintained for back compatibility\n"; + print STDERR " but preferably use -decoder-parameters)\n"; + exit(1); +} + + + + + +#printparameters +sub print_parameters(){ + print STDERR "Inputfile: $inputlist\n"; + print STDERR "Configuration file: $cfgfile\n"; + print STDERR "Decoder in use: $mosescmd\n"; + print STDERR "Number of jobs:$jobs\n"; + print STDERR "Nbest list: $nbestlist\n" if ($nbestflag); + print STDERR "Output Search Graph: $searchgraphlist\n" if ($searchgraphflag); + print STDERR "Output Word Graph: $wordgraphlist\n" if ($wordgraphflag); + print STDERR "LogFile:$logfile\n" if ($logflag); + print STDERR "Qsub name: $qsubname\n"; + print STDERR "Queue parameters: $queueparameters\n"; + print STDERR "Inputtype: text\n" if $inputtype == 0; + print STDERR "Inputtype: confusion network\n" if $inputtype == 1; + print STDERR "Inputtype: lattices\n" if $inputtype == 2; + + print STDERR "parameters directly passed to Moses: $mosesparameters\n"; +} + +#get parameters for log file +sub getLogParameters(){ + if ($logfile){ $logflag=1; } +} + +#get parameters for nbest computation (possibly from configuration file) +sub getNbestParameters(){ + if (!$nbestlist){ + open (CFG, "$cfgfile"); + while (chomp($_=)){ + if (/^\[n-best-list\]/){ + my $tmp; + while (chomp($tmp=)){ + last if $tmp eq "" || $tmp=~/^\[/; + $nbestlist .= "$tmp "; + } + last; + } + } + close(CFG); + } + + if ($nbestlist){ + if ($oldnbestfile){ + print STDERR "There is a conflict between NEW parameter -n-best-list and OBSOLETE parameter -n-best-file\n"; + print STDERR "Please use only -nbest-list ' [distinct]\n"; + exit; + } + } + else{ + if ($oldnbestfile){ + print STDERR "You are using the OBSOLETE parameter -n-best-file\n"; + print STDERR "Next time please use only -n-best-list ' [distinct]\n"; + $nbestlist="$oldnbestfile"; + if ($oldnbest){ $nbestlist.=" $oldnbest"; } + else { $nbestlist.=" 1"; } + } + } + + if ($nbestlist){ + my @tmp=split(/[ \t]+/,$nbestlist); + @nbestlist = @tmp; + + if ($nbestlist[0] eq '-'){ $nbestfile="nbest"; } + else{ chomp($nbestfile=`basename $nbestlist[0]`); } + $nbestflag=1; + } + print STDERR "getNbest\n"; + print STDERR "nbestflag = $nbestflag\n"; + print STDERR "nbestfile = $nbestfile\n"; + + +} + +#get parameters for search graph computation (possibly from configuration file) +sub getSearchGraphParameters(){ + if (!$searchgraphlist){ + open (CFG, "$cfgfile"); + while (chomp($_=)){ + if (/^\[output-search-graph\]/ || /^\[osg\]/){ + my $tmp; + while (chomp($tmp=)){ + last if $tmp eq "" || $tmp=~/^\[/; + $searchgraphlist = "$tmp"; + } + last; + } + } + close(CFG); + } + if ($searchgraphlist){ + if ($searchgraphlist eq '-'){ $searchgraphfile="searchgraph"; } + else{ chomp($searchgraphfile=`basename $searchgraphlist`); } + $searchgraphflag=1; + } +} + +#get parameters for word graph computation (possibly from configuration file) +sub getWordGraphParameters(){ + if (!$wordgraphlist){ + open (CFG, "$cfgfile"); + while (chomp($_=)){ + if (/^\[output-word-graph\]/ || /^\[owg\]/){ + my $tmp; + while (chomp($tmp=)){ + last if $tmp eq "" || $tmp=~/^\[/; + $wordgraphlist .= "$tmp "; + } + last; + } + } + close(CFG); + } + if ($wordgraphlist){ + my @tmp=split(/[ \t]+/,$wordgraphlist); + @wordgraphlist = @tmp; + + if ($wordgraphlist[0] eq '-'){ $wordgraphfile="wordgraph"; } + else{ chomp($wordgraphfile=`basename $wordgraphlist[0]`); } + $wordgraphflag=1; + } +} + +sub sanity_check_order_of_lambdas { + my $featlist = shift; + my $filename_or_stream = shift; + + my @expected_lambdas = @{$featlist->{"names"}}; + my @got = get_order_of_scores_from_nbestlist($filename_or_stream); + die "Mismatched lambdas. Decoder returned @got, we expected @expected_lambdas" + if "@got" ne "@expected_lambdas"; +} + + + + +####################### +#Script starts here + +init(); + +print "I have started parallel moses!!"; + +# moses.ini file uses FULL names for lambdas, while this training script +# internally (and on the command line) uses ABBR names. +my @ABBR_FULL_MAP = qw(d=weight-d lm=weight-l tm=weight-t w=weight-w + g=weight-generation lex=weight-lex I=weight-i); +my %ABBR2FULL = map {split/=/,$_,2} @ABBR_FULL_MAP; +my %FULL2ABBR = map {my ($a, $b) = split/=/,$_,2; ($b, $a);} @ABBR_FULL_MAP; + + + + +version() if $version; +usage() if $help; + +####################################### +# incorporate run_decoder() here + +### moved to below +# my $qsubname = mert$run; +# $mosesparameters = "$___DECODER_FLAGS $decoder_config"; +# $nbestlist = "$filename $___N_BEST_LIST_SIZE"; + + +my $featlist = get_featlist_from_moses("./run$run.moses.ini"); +$featlist = insert_ranges_to_featlist($featlist, $___RANGES); + + +## sub run_decoder { +# my ($featlist, $run, $need_to_normalize) = @_; +my $filename_template = "run%d.best$___N_BEST_LIST_SIZE.out"; +my $filename = sprintf($filename_template, $run); +my $lsamp_filename = undef; +if ($___LATTICE_SAMPLES) { + my $lsamp_filename_template = "run%d.lsamp$___LATTICE_SAMPLES.out"; + $lsamp_filename = sprintf($lsamp_filename_template, $run); +} + +# user-supplied parameters +print STDERR "params = $___DECODER_FLAGS\n"; + + +# parameters to set all model weights (to override moses.ini) +my @vals = @{$featlist->{"values"}}; +if ($need_to_normalize) { + print STDERR "Normalizing lambdas: @vals\n"; + my $totlambda=0; + grep($totlambda+=abs($_),@vals); + grep($_/=$totlambda,@vals); +} + + +######################################## + + # parameters to set all model weights (to override moses.ini) + my @vals = @{$featlist->{"values"}}; + if ($need_to_normalize) { + print STDERR "Normalizing lambdas: @vals\n"; + my $totlambda=0; + grep($totlambda+=abs($_),@vals); + grep($_/=$totlambda,@vals); + } + # moses now does not seem accept "-tm X -tm Y" but needs "-tm X Y" + my %model_weights; + for(my $i=0; $i{"names"}}); $i++) { + my $name = $featlist->{"names"}->[$i]; + $model_weights{$name} = "-$name" if !defined $model_weights{$name}; + $model_weights{$name} .= sprintf " %.6f", $vals[$i]; + } + my $decoder_config = join(" ", values %model_weights); + $decoder_config .= " -weight-file run$run.sparse-weights" if -e "run$run.sparse-weights"; + print STDERR "DECODER_CFG = $decoder_config\n"; + print "decoder_config = $decoder_config\n"; + + + + +######################################### + +# moses now does not seem accept "-tm X -tm Y" but needs "-tm X Y" +my %model_weights; +for(my $i=0; $i{"names"}}); $i++) { + my $name = $featlist->{"names"}->[$i]; + $model_weights{$name} = "-$name" if !defined $model_weights{$name}; + $model_weights{$name} .= sprintf " %.6f", $vals[$i]; +} +my $decoder_config = join(" ", values %model_weights); +$decoder_config .= " -weight-file run$run.sparse-weights" if -e "run$run.sparse-weights"; +print STDERR "DECODER_CFG = $decoder_config\n"; +print STDERR "decoder_config = $decoder_config\n"; + +####### moved here??? ########### +$qsubname = "dec$run"; +# $mosesparameters = "$___DECODER_FLAGS $decoder_config"; +$mosesparameters = "$___DECODER_FLAGS $decoder_config "; + +print STDERR "moses parameter with ___DECODER_FLAGS $___DECODER_FLAGS decoder_config $decoder_config and $qsubname\n"; +print STDERR "$mosesparameters\n"; + +$nbestlist = "$filename $___N_BEST_LIST_SIZE"; + +print STDERR "to output -n-best-list $nbestlist\n"; + +init_secondpart(); +################################# + + +# run the decoder +my $decoder_cmd; +my $lsamp_cmd = ""; +if ($___LATTICE_SAMPLES) { + $lsamp_cmd = " -lattice-samples $lsamp_filename $___LATTICE_SAMPLES "; +} + + +if (!defined $inputlist || !defined $mosescmd || ! defined $cfgfile) { + print STDERR "Please specify -input-file, -decoder and -config\n"; + usage(); +} + +#checking if inputfile exists +if (! -e ${inputlist} ){ + print STDERR "Inputfile ($inputlist) does not exists\n"; + usage(); +} + +#checking if decoder exists +if (! -e $mosescmd) { + print STDERR "Decoder ($mosescmd) does not exists\n"; + usage(); +} + +#checking if configfile exists +if (! -e $cfgfile) { + print STDERR "Configuration file ($cfgfile) does not exists\n"; + usage(); +} + + +print_parameters(); # so that people know +exit(1) if $dbg; # debug mode: just print and do not run + + +#splitting test file in several parts +#$decimal="-d"; #split does not accept this options (on MAC OS) +my $decimal=""; + +my $cmd; +my $sentenceN; +my $splitN; + +my @idxlist=(); +my $idxliststr=""; + +if ($inputtype==0){ #text input +#getting the number of input sentences (one sentence per line) + chomp($sentenceN=`wc -l ${inputlist} | awk '{print \$1}' `); + +#Reducing the number of jobs if less sentences to translate + if ($jobs>$sentenceN){ $jobs=$sentenceN; } + +#Computing the number of sentences for each files + if ($sentenceN % $jobs == 0){ $splitN=int($sentenceN / $jobs); } + else{ $splitN=int($sentenceN /$jobs) + 1; } + + if ($dbg){ + print STDERR "There are $sentenceN sentences to translate\n"; + print STDERR "There are at most $splitN sentences per job\n"; + } + + $cmd="split $decimal -a 2 -l $splitN $inputlist ${inputfile}.$splitpfx-"; + safesystem("$cmd") or die; +} +elsif ($inputtype==1){ #confusion network input + my $tmpfile="/tmp/cnsplit$$"; + $cmd="cat $inputlist | perl -pe 's/\\n/ _CNendline_ /g;' | perl -pe 's/_CNendline_ _CNendline_ /_CNendline_\\n/g;' > $tmpfile"; + safesystem("$cmd") or die; + +#getting the number of input CNs + chomp($sentenceN=`wc -l $tmpfile | awk '{print \$1}' `); + +#Reducing the number of jobs if less CNs to translate + if ($jobs>$sentenceN){ $jobs=$sentenceN; } + +#Computing the number of CNs for each files + if ($sentenceN % $jobs == 0){ $splitN=int($sentenceN / $jobs); } + else{ $splitN=int($sentenceN /$jobs) + 1; } + + if ($dbg){ + print STDERR "There are $sentenceN confusion networks to translate\n"; + print STDERR "There are at most $splitN sentences per job\n"; + } + + $cmd="split $decimal -a 2 -l $splitN $tmpfile $tmpfile-"; + safesystem("$cmd") or die; + + my @idxlist=(); + chomp(@idxlist=`ls $tmpfile-*`); + grep(s/.+(\-\S+)$/$1/e,@idxlist); + + foreach my $idx (@idxlist){ + $cmd="perl -pe 's/ _CNendline_ /\\n/g;s/ _CNendline_/\\n/g;'"; + safesystem("cat $tmpfile$idx | $cmd > ${inputfile}.$splitpfx$idx ; \\rm -f $tmpfile$idx;"); + } +} +elsif ($inputtype==2){ #confusion network input +#getting the number of input lattices (one lattice per line) + chomp($sentenceN=`wc -l ${inputlist} | awk '{print \$1}' `); + +#Reducing the number of jobs if less lattices to translate + if ($jobs>$sentenceN){ $jobs=$sentenceN; } + +#Computing the number of sentences for each files + if ($sentenceN % $jobs == 0){ $splitN=int($sentenceN / $jobs); } + else{ $splitN=int($sentenceN /$jobs) + 1; } + + if ($dbg){ + print STDERR "There are $sentenceN lattices to translate\n"; + print STDERR "There are at most $splitN lattices per job\n"; + } + + $cmd="split $decimal -a 2 -l $splitN $inputlist ${inputfile}.$splitpfx-"; + safesystem("$cmd") or die; +} +else{ #unknown input type + die "INPUTTYPE:$inputtype is unknown!\n"; +} + +chomp(@idxlist=`ls ${inputfile}.$splitpfx-*`); +grep(s/.+(\-\S+)$/$1/e,@idxlist); + +safesystem("mkdir -p $tmpdir") or die; + +preparing_script(); + + + +#launching process through the queue +my @sgepids =(); +my $splitdecodejid=""; + +my @idx_todo = (); +foreach (@idxlist) { push @idx_todo,$_; } + +# loop up to --robust times +my $max_robust = $robust; +my $robust_idx; +while ($robust && scalar @idx_todo) { + $robust--; + + my $failure=0; + + + foreach my $idx (@idx_todo){ + + if ($old_sge) { + # old SGE understands -b no as the default and does not understand 'yes' + $batch_and_join = "-j y"; + } else { + $batch_and_join = "-b yes -j yes"; # -b yes to submit bash script + } + + + ##### Replace the direct qsub command with submit_or_exec_thu_host() ############# + + my $split_decoder_cmd = "${jobscript}${idx}.bash"; + &submit_or_exec_thu_host($submithost,$run,$idx,$split_decoder_cmd,$batch_and_join,"decode$run$idx.out","decode$run$idx.err","decode$run$idx.id"); + chomp($jobid=`tail -n 1 decode$run$idx.id`); + print STDERR "JOBID for decoding sub-task decode$run$idx is $jobid\n"; + + ################################################################################### + + ##### # get jobid ################################################################# + push @sgepids, $jobid; + $splitdecodejid .= " $jobid"; + ################################################################################### + } + + ## clear temp file for this robust trial + foreach my $idx (@idx_todo){ + &exit_submit_thu_host($submithost,$run,$idx,"","decode$run$idx.out","decode$run$idx.err","decode$run$idx.id","decode$run$idx.id.pid",$splitdecodejid); + } + + ## WAIT JOB for robust iteration + $cmd = "date"; + $robust_idx = $max_robust - $robust; + &submit_or_exec_thu_host($submithost,$run,"R${robust_idx}.W",$cmd,"-sync y","decode${run}R${robust_idx}.W.out","decode${run}R${robust_idx}.W.err","decode${run}R${robust_idx}.W.id",$splitdecodejid); + # no need to harvest jobid as this is in sync mode + + # clear up tmp files + &exit_submit_thu_host($submithost,$run,"R${robust_idx}.W","","decode${run}R${robust_idx}.W.out","decode${run}R${robust_idx}.W.err","decode${run}R${robust_idx}.W.id","decode${run}R${robust_idx}.W.id.pid",$splitdecodejid); + + + + # check if some translations failed + my @idx_still_todo = check_translation(); + if ($robust) { + # if robust, redo crashed jobs + ##RESUBMIT_ANYWAY## if ((scalar @idx_still_todo) == (scalar @idxlist)) { + ##RESUBMIT_ANYWAY## # ... but not if all crashed + ##RESUBMIT_ANYWAY## print STDERR "everything crashed, not trying to resubmit jobs\n"; + ##RESUBMIT_ANYWAY## $robust = 0; + ##RESUBMIT_ANYWAY## kill_all_and_quit(); + ##RESUBMIT_ANYWAY## } + @idx_todo = @idx_still_todo; + } + else { + if (scalar (@idx_still_todo)) { + print STDERR "some jobs crashed: ".join(" ",@idx_still_todo)."\n"; + # kill_all_and_quit(); + } + + } + +} + + + +$idxliststr = join(" ",@idxlist); + + +$postdecodecmd = "$SCRIPTS_ROOTDIR/training/sge-nosync/moses-parallel-postdecode-sge-nosync.pl" if !defined $postdecodecmd; + +$postdecodeargs = "" if !defined $postdecodeargs; +$postdecodeargs = "$postdecodeargs --process-id $$ --idxliststr \"$idxliststr\""; +$postdecodeargs = "$postdecodeargs --nbestfile $nbestfile" if (defined $nbestfile); +$postdecodeargs = "$postdecodeargs --outnbest $outnbest" if (defined $outnbest); +$postdecodeargs = "$postdecodeargs --input-file $inputfile" if ($inputfile); + +my $cmd = "$postdecodecmd $postdecodeargs"; +&submit_or_exec_thu_host($submithost,$run,".CONCAT",$cmd,"","run$run.out","run$run.err","postdecode$run.id","$splitdecodejid"); + + +chomp($jobid=`tail -n 1 postdecode$run.id`); +$prevjid = $jobid; + +## clear up tmp +&exit_submit_thu_host($submithost,$run,".CONCAT","","run$run.out","run$run.err","postdecode$run.id","postdecode$run.id.pid",$prevjid); + + + +##### OVERALL WAIT JOB ########## + my $syncscript = "${jobscript}.waitall.sh"; + + open (OUT, ">$syncscript"); + my $scriptheader="\#\!/bin/bash\n\#\$ -S /bin/sh\n# Both lines are needed to invoke base\n#the above line is ignored by qsub, unless parameter \"-b yes\" is set!\n\n"; + $scriptheader .="uname -a\n\n"; + $scriptheader .="cd $___WORKING_DIR\n\n"; + print OUT $scriptheader; + print OUT "'date'"; + close(OUT); + + # safesystem("echo 'date' > $syncscript") or kill_all_and_quit(); + chmod(oct(755),"$syncscript"); + + # $cmd="qsub $queueparameters -o /dev/null -e /dev/null -N $qsubname.W -b y /bin/ls > $qsubname.W.log"; + $batch_and_join = "-j y"; + + &submit_or_exec_thu_host($submithost,$run,".W",$syncscript,$batch_and_join,"decode$run.W.out","decode$run.W.err","decode$run.W.id",$prevjid); + chomp($jobid=`tail -n 1 decode$run.W.id`); + print STDERR "JOBID for wait-job for all decoding sub-task is $jobid\n"; + $prevjid = $jobid; + + ## clear-up tmp + &exit_submit_thu_host($submithost,$run,".W",$batch_and_join,"decode$run.W.out","decode$run.W.err","decode$run.W.id","decode$run.W.id.pid",$prevjid); + +exit(); + + + + + +### ending scripts in run_decoder() ############## +sanity_check_order_of_lambdas($featlist, $filename); +return ($filename, $lsamp_filename); +################################################## + + + + +#script creation +sub preparing_script(){ + my $currStartTranslationId = 0; + + foreach my $idx (@idxlist){ + my $scriptheader=""; + $scriptheader.="\#\! /bin/bash\n\n"; + # !!! this is useless. qsub ignores the first line of the script. + # Pass '-S /bin/bash' to qsub instead. + $scriptheader.="uname -a\n\n"; + $scriptheader.="ulimit -c 0\n\n"; # avoid coredumps + $scriptheader.="cd $workingdir\n\n"; + + open (OUT, "> ${jobscript}${idx}.bash"); + print OUT $scriptheader; + my $inputmethod = $feed_moses_via_stdin ? "<" : "-input-file"; + + my $tmpnbestlist=""; + if ($nbestflag){ + $tmpnbestlist="$tmpdir/$nbestfile.$splitpfx$idx $nbestlist[1]"; + $tmpnbestlist = "$tmpnbestlist $nbestlist[2]" if scalar(@nbestlist)==3; + $tmpnbestlist = "-n-best-list $tmpnbestlist"; + } + + $outnbest=$nbestlist[0]; + if ($nbestlist[0] eq '-'){ $outnbest="nbest$$"; } + + print STDERR "n-best-list $tmpnbestlist\n"; + + + my $tmpalioutfile = ""; + if (defined $alifile){ + $tmpalioutfile="-alignment-output-file $tmpdir/$alifile.$splitpfx$idx"; + } + + my $tmpsearchgraphlist=""; + if ($searchgraphflag){ + $tmpsearchgraphlist="-output-search-graph $tmpdir/$searchgraphfile.$splitpfx$idx"; + } + + my $tmpwordgraphlist=""; + if ($wordgraphflag){ + $tmpwordgraphlist="-output-word-graph $tmpdir/$wordgraphfile.$splitpfx$idx $wordgraphlist[1]"; + } + + my $tmpStartTranslationId = ""; # "-start-translation-id $currStartTranslationId"; + + print OUT "$mosescmd $mosesparameters $tmpStartTranslationId $tmpalioutfile $tmpwordgraphlist $tmpsearchgraphlist $tmpnbestlist $inputmethod ${inputfile}.$splitpfx$idx > $tmpdir/${inputfile}.$splitpfx$idx.trans\n\n"; + print OUT "echo exit status \$\?\n\n"; + + if (defined $alifile){ + print OUT "\\mv -f $tmpdir/${alifile}.$splitpfx$idx .\n\n"; + print OUT "echo exit status \$\?\n\n"; + } + if ($nbestflag){ + print OUT "\\mv -f $tmpdir/${nbestfile}.$splitpfx$idx .\n\n"; + print OUT "echo exit status \$\?\n\n"; + } + if ($searchgraphflag){ + print OUT "\\mv -f $tmpdir/${searchgraphfile}.$splitpfx$idx .\n\n"; + print OUT "echo exit status \$\?\n\n"; + } + + if ($wordgraphflag){ + print OUT "\\mv -f $tmpdir/${wordgraphfile}.$splitpfx$idx .\n\n"; + print OUT "echo exit status \$\?\n\n"; + } + + print OUT "\\mv -f $tmpdir/${inputfile}.$splitpfx$idx.trans .\n\n"; + print OUT "echo exit status \$\?\n\n"; + close(OUT); + + #setting permissions of each script + chmod(oct(755),"${jobscript}${idx}.bash"); + + $currStartTranslationId += $splitN; + } +} + +sub concatenate_wordgraph(){ + my $oldcode=""; + my $newcode=-1; + my %inplength = (); + my $offset = 0; + + my $outwordgraph=$wordgraphlist[0]; + if ($wordgraphlist[0] eq '-'){ $outwordgraph="wordgraph$$"; } + + open (OUT, "> $outwordgraph"); + foreach my $idx (@idxlist){ + +#computing the length of each input file + my @in=(); + open (IN, "${inputfile}.${splitpfx}${idx}.trans"); + @in=; + close(IN); + $inplength{$idx} = scalar(@in); + + open (IN, "${wordgraphfile}.${splitpfx}${idx}"); + while (){ + + my $code=""; + if (/^UTTERANCE=/){ + ($code)=($_=~/^UTTERANCE=(\d+)/); + + print STDERR "code:$code offset:$offset\n"; + $code += $offset; + if ($code ne $oldcode){ + +# if there is a jump between two consecutive codes +# it means that an input sentence is not translated +# fill this hole with a "fictitious" list of wordgraphs +# comprising just one "_EMPTYSEARCHGRAPH_ + while ($code - $oldcode > 1){ + $oldcode++; + print OUT "UTTERANCE=$oldcode\n"; + print STDERR " to OUT -> code:$oldcode\n"; + print OUT "_EMPTYWORDGRAPH_\n"; + } + } + + $oldcode=$code; + print OUT "UTTERANCE=$oldcode\n"; + next; + } + print OUT "$_"; + } + close(IN); + $offset += $inplength{$idx}; + + while ($offset - $oldcode > 1){ + $oldcode++; + print OUT "UTTERANCE=$oldcode\n"; + print OUT "_EMPTYWORDGRAPH_\n"; + } + } + close(OUT); +} + + +sub concatenate_searchgraph(){ + my $oldcode=""; + my $newcode=-1; + my %inplength = (); + my $offset = 0; + + my $outsearchgraph=$searchgraphlist; + if ($searchgraphlist eq '-'){ $outsearchgraph="searchgraph$$"; } + + open (OUT, "> $outsearchgraph"); + foreach my $idx (@idxlist){ + +#computing the length of each input file + my @in=(); + open (IN, "${inputfile}.${splitpfx}${idx}.trans"); + @in=; + close(IN); + $inplength{$idx} = scalar(@in); + + open (IN, "${searchgraphfile}.${splitpfx}${idx}"); + while (){ + my ($code,@extra)=split(/[ \t]+/,$_); + $code += $offset; + if ($code ne $oldcode){ + +# if there is a jump between two consecutive codes +# it means that an input sentence is not translated +# fill this hole with a "fictitious" list of searchgraphs +# comprising just one "_EMPTYSEARCHGRAPH_ + while ($code - $oldcode > 1){ + $oldcode++; + print OUT "$oldcode _EMPTYSEARCHGRAPH_\n"; + } + } + $oldcode=$code; + print OUT join(" ",($oldcode,@extra)); + } + close(IN); + $offset += $inplength{$idx}; + + while ($offset - $oldcode > 1){ + $oldcode++; + print OUT "$oldcode _EMPTYSEARCHGRAPH_\n"; + } + } + close(OUT); +} + +sub concatenate_nbest(){ + my $oldcode=""; + my $newcode=-1; + my %inplength = (); + my $offset = 0; + +# get the list of feature and set a fictitious string with zero scores + open (IN, "${nbestfile}.${splitpfx}$idxlist[0]"); + my $str = ; + chomp($str); + close(IN); + my ($code,$trans,$featurescores,$globalscore)=split(/\|\|\|/,$str); + + my $emptytrans = " "; + my $emptyglobalscore = " 0.0"; + my $emptyfeaturescores = $featurescores; + $emptyfeaturescores =~ s/[-0-9\.]+/0/g; + + my $outnbest=$nbestlist[0]; + if ($nbestlist[0] eq '-'){ $outnbest="nbest$$"; } + + open (OUT, "> $outnbest"); + foreach my $idx (@idxlist){ + +#computing the length of each input file + my @in=(); + open (IN, "${inputfile}.${splitpfx}${idx}.trans"); + @in=; + close(IN); + $inplength{$idx} = scalar(@in); + + open (IN, "${nbestfile}.${splitpfx}${idx}"); + while (){ + my ($code,@extra)=split(/\|\|\|/,$_); + $code += $offset; + if ($code ne $oldcode){ + +# if there is a jump between two consecutive codes +# it means that an input sentence is not translated +# fill this hole with a "fictitious" list of translation +# comprising just one "emtpy translation" with zero scores + while ($code - $oldcode > 1){ + $oldcode++; + print OUT join("\|\|\|",($oldcode,$emptytrans,$emptyfeaturescores,$emptyglobalscore)),"\n"; + } + } + $oldcode=$code; + print OUT join("\|\|\|",($oldcode,@extra)); + } + close(IN); + $offset += $inplength{$idx}; + + while ($offset - $oldcode > 1){ + $oldcode++; + print OUT join("\|\|\|",($oldcode,$emptytrans,$emptyfeaturescores,$emptyglobalscore)),"\n"; + } + } + close(OUT); +} + + + +sub check_exit_status(){ + print STDERR "check_exit_status\n"; + my $failure=0; + foreach my $idx (@idxlist){ + print STDERR "check_exit_status of job $idx\n"; + open(IN,"$qsubout$idx"); + while (){ + $failure=1 if (/exit status 1/); + } + close(IN); + } + return $failure; +} + +sub kill_all_and_quit(){ + print STDERR "Got interrupt or something failed.\n"; + print STDERR "kill_all_and_quit\n"; + foreach my $id (@sgepids){ + print STDERR "qdel $id\n"; + safesystem("qdel $id"); + } + + print STDERR "Translation was not performed correctly\n"; + print STDERR "or some of the submitted jobs died.\n"; + print STDERR "qdel function was called for all submitted jobs\n"; + + exit(1); +} + + +sub check_translation(){ + #checking if all sentences were translated + my $inputN; + my $outputN; + my @failed = (); + foreach my $idx (@idx_todo){ + if ($inputtype==0){#text input + chomp($inputN=`wc -l ${inputfile}.$splitpfx$idx | cut -d' ' -f1`); + } + elsif ($inputtype==1){#confusion network input + chomp($inputN=`cat ${inputfile}.$splitpfx$idx | perl -pe 's/\\n/ _CNendline_ /g;' | perl -pe 's/_CNendline_ _CNendline_ /_CNendline_\\n/g;' | wc -l | cut -d' ' -f1 `); + } + elsif ($inputtype==2){#lattice input + chomp($inputN=`wc -l ${inputfile}.$splitpfx$idx | cut -d' ' -f1`); + } + else{#unknown input + die "INPUTTYPE:$inputtype is unknown!\n"; + } + chomp($outputN=`wc -l ${inputfile}.$splitpfx$idx.trans | cut -d' ' -f1`); + + if ($inputN != $outputN){ + print STDERR "Split ($idx) were not entirely translated\n"; + print STDERR "outputN=$outputN inputN=$inputN\n"; + print STDERR "outputfile=${inputfile}.$splitpfx$idx.trans inputfile=${inputfile}.$splitpfx$idx\n"; + push @failed,$idx; + } + } + return @failed; +} + +sub check_translation_old_sge(){ + #checking if all sentences were translated + my $inputN; + my $outputN; + foreach my $idx (@idx_todo){ + if ($inputtype==0){#text input + chomp($inputN=`wc -l ${inputfile}.$splitpfx$idx | cut -d' ' -f1`); + } + elsif ($inputtype==1){#confusion network input + chomp($inputN=`cat ${inputfile}.$splitpfx$idx | perl -pe 's/\\n/ _CNendline_ /g;' | perl -pe 's/_CNendline_ _CNendline_ /_CNendline_\\n/g;' | wc -l | + cut -d' ' -f1 `); + } + elsif ($inputtype==2){#lattice input + chomp($inputN=`wc -l ${inputfile}.$splitpfx$idx | cut -d' ' -f1`); + } + else{#unknown input + die "INPUTTYPE:$inputtype is unknown!\n"; + } + chomp($outputN=`wc -l ${inputfile}.$splitpfx$idx.trans | cut -d' ' -f1`); + + if ($inputN != $outputN){ + print STDERR "Split ($idx) were not entirely translated\n"; + print STDERR "outputN=$outputN inputN=$inputN\n"; + print STDERR "outputfile=${inputfile}.$splitpfx$idx.trans inputfile=${inputfile}.$splitpfx$idx\n"; + return 1; + } + + } + return 0; +} + +sub remove_temporary_files(){ + #removing temporary files + foreach my $idx (@idxlist){ + unlink("${inputfile}.${splitpfx}${idx}.trans"); + unlink("${inputfile}.${splitpfx}${idx}"); + if (defined $alifile){ unlink("${alifile}.${splitpfx}${idx}"); } + if ($nbestflag){ unlink("${nbestfile}.${splitpfx}${idx}"); } + if ($searchgraphflag){ unlink("${searchgraphfile}.${splitpfx}${idx}"); } + if ($wordgraphflag){ unlink("${wordgraphfile}.${splitpfx}${idx}"); } + unlink("${jobscript}${idx}.bash"); + unlink("${jobscript}${idx}.log"); + unlink("$qsubname.W.log"); + unlink("$qsubout$idx"); + unlink("$qsuberr$idx"); + rmdir("$tmpdir"); + } + if ($nbestflag && $nbestlist[0] eq '-'){ unlink("${nbestfile}$$"); }; + if ($searchgraphflag && $searchgraphlist eq '-'){ unlink("${searchgraphfile}$$"); }; + if ($wordgraphflag && $wordgraphlist eq '-'){ unlink("${wordgraphfile}$$"); }; +} + +sub safesystem { + print STDERR "Executing: @_\n"; + system(@_); + if ($? == -1) { + print STDERR "Failed to execute: @_\n $!\n"; + exit(1); + } + elsif ($? & 127) { + printf STDERR "Execution of: @_\n died with signal %d, %s coredump\n", + ($? & 127), ($? & 128) ? 'with' : 'without'; + exit 1; + } + else { + my $exitcode = $? >> 8; + print STDERR "Exit code: $exitcode\n" if $exitcode; + return ! $exitcode; + } +} + + +# look for the correct pwdcmd (pwd by default, pawd if it exists) +# I assume that pwd always exists +sub getPwdCmd(){ + my $pwdcmd="pwd"; + my $a; + chomp($a=`which pawd | head -1 | awk '{print $1}'`); + if ($a && -e $a){ $pwdcmd=$a; } + return $pwdcmd; +} + + + +sub get_featlist_from_moses { + # run moses with the given config file and return the list of features and + # their initial values + + ### variable conversion for moses-parallel-sge-nosync.pl ### + my $___INPUTTYPE = $inputtype; + my $___DECODER = $mosescmd; + ###################################################### + my $configfn = shift; + + + # forceful load features list every time + my $featlistfn = "./features.list.run${run}_start"; + if (-e $featlistfn) { + print STDERR "Removing old features list: $featlistfn\n"; + print STDERR "Generating a new one with $configfn\n"; + } + print STDERR "Asking moses for feature names and values from $configfn\n"; + my $cmd = "$___DECODER $___DECODER_FLAGS -config $configfn -inputtype $___INPUTTYPE -show-weights > $featlistfn"; + print STDERR "$cmd\n"; #DEBUG + safesystem($cmd) or die "Failed to run moses with the config $configfn"; + + + # read feature list + my @names = (); + my @startvalues = (); + open(INI,$featlistfn) or die "Can't read $featlistfn"; + my $nr = 0; + my @errs = (); + while () { + $nr++; + chomp; + /^(.+) (\S+) (\S+)$/ || die("invalid feature: $_"); + my ($longname, $feature, $value) = ($1,$2,$3); + next if $value eq "sparse"; + push @errs, "$featlistfn:$nr:Bad initial value of $feature: $value\n" + if $value !~ /^[+-]?[0-9.e]+$/; + push @errs, "$featlistfn:$nr:Unknown feature '$feature', please add it to \@ABBR_FULL_MAP\n" + if !defined $ABBR2FULL{$feature}; + push @names, $feature; + push @startvalues, $value; + } + close INI; + if (scalar @errs) { + print STDERR join("", @errs); + exit 1; + } + return {"names"=>\@names, "values"=>\@startvalues}; +} + + +sub insert_ranges_to_featlist { + ### variable conversion for moses-parallel-sge-nosync.pl ### + my $___INPUTTYPE = $inputtype; + my $___DECODER = $mosescmd; + ###################################################### + + my $featlist = shift; + my $ranges = shift; + + $ranges = [] if !defined $ranges; + + # first collect the ranges from options + my $niceranges; + foreach my $range (@$ranges) { + my $name = undef; + foreach my $namedpair (split /,/, $range) { + if ($namedpair =~ /^(.*?):/) { + $name = $1; + $namedpair =~ s/^.*?://; + die "Unrecognized name '$name' in --range=$range" + if !defined $ABBR2FULL{$name}; + } + my ($min, $max) = split /\.\./, $namedpair; + die "Bad min '$min' in --range=$range" if $min !~ /^-?[0-9.]+$/; + die "Bad max '$max' in --range=$range" if $min !~ /^-?[0-9.]+$/; + die "No name given in --range=$range" if !defined $name; + push @{$niceranges->{$name}}, [$min, $max]; + } + } + + # now populate featlist + my $seen = undef; + for(my $i=0; $i{"names"}}); $i++) { + my $name = $featlist->{"names"}->[$i]; + $seen->{$name} ++; + my $min = 0.0; + my $max = 1.0; + if (defined $niceranges->{$name}) { + my $minmax = shift @{$niceranges->{$name}}; + ($min, $max) = @$minmax if defined $minmax; + } + $featlist->{"mins"}->[$i] = $min; + $featlist->{"maxs"}->[$i] = $max; + } + return $featlist; +} + + + + +sub submit_or_exec_thu_host { + # use Net::OpenSSH::Compat::Perl; + + my $argvlen = @_; + my $submithost = undef; + my $run = -1; + my $idx = ""; + my $batch_and_join = ""; + my $my_username = undef; + my $cmd = undef; + my $qsubwrapcmd = undef; + my $stdout = undef; + my $stderr = undef; + my $jidfile = undef; + my $prevjid = undef; + + + # if supply 7 arguments, exec without submit + # if supply 8 arguments, then submit new job + # if supply 9 arguments, wait for the previous job to finish + if ($argvlen == 7){ + ($submithost,$run,$idx,$cmd,$batch_and_join,$stdout,$stderr) = @_; + } elsif ($argvlen == 8){ + ($submithost,$run,$idx,$cmd,$batch_and_join,$stdout,$stderr,$jidfile) = @_; + } elsif ($argvlen == 9){ + ($submithost,$run,$idx,$cmd,$batch_and_join,$stdout,$stderr,$jidfile,$prevjid) = @_; + } + + + + chomp(my $my_username = `whoami`); + my $ssh = Net::OpenSSH::Compat::Perl->new($submithost, debug=>0); + + print STDERR "submithost = $submithost\n"; + print STDERR "qusbwrapper at = $qsubwrapper\n"; + + $ssh->login(); + + if ($argvlen == 9) { + $qsubwrapcmd = "$qsubwrapper -command='$cmd' -queue-parameter=\"$queueparameters $batch_and_join\" -qsub-prefix='$qsubname$idx' -stdout=$stdout -stderr=$stderr -jidfile=$jidfile -prevjid='$prevjid'"; + } elsif ($argvlen == 8) { + # $qsubwrapcmd = "$qsubwrapper -command='${jobscript}${idx}.bash' -queue-parameter=\"$queueparameters $batch_and_join\" -qsub-prefix='$qsubname$idx' -stdout=$stdout -stderr=$stderr -jidfile=$jidfile"; + $qsubwrapcmd = "$qsubwrapper -command='$cmd' -queue-parameter=\"$queueparameters $batch_and_join\" -qsub-prefix='$qsubname$idx' -stdout=$stdout -stderr=$stderr -jidfile=$jidfile"; + } + print STDERR "Executing $qsubwrapcmd in $___WORKING_DIR\n"; + $ssh->cmd("cd $___WORKING_DIR && $qsubwrapcmd"); + +} + +sub exit_submit_thu_host { + + my $argvlen = @_; + my $submithost = undef; + my $run = -1; + my $idx = ""; + my $batch_and_join = ""; + my $my_username = undef; + my $cmd = undef; + my $stdout = undef; + my $stderr = undef; + my $jidfile = undef; + my $pidfile = undef; + my $prevjid = undef; + my $prevjidarraysize = 0; + my @prevjidarray = (); + my $pid = undef; + my $qsubcmd=""; + my $hj=""; + + # if supply 8 arguments, then submit new job + # if supply 9 arguments, wait for the previous job to finish + if ($argvlen == 6){ + ($submithost,$run,$idx,$batch_and_join,$stdout,$stderr) = @_; + } elsif ($argvlen == 8){ + ($submithost,$run,$idx,$batch_and_join,$stdout,$stderr,$jidfile,$pidfile) = @_; + } elsif ($argvlen == 9){ + ($submithost,$run,$idx,$batch_and_join,$stdout,$stderr,$jidfile,$pidfile,$prevjid) = @_; + } + + # parse prevjid ######################## + $prevjid =~ s/^\s+|\s+$//g; + @prevjidarray = split(/\s+/,$prevjid); + $prevjidarraysize = scalar(@prevjidarray); + ######################################## + + + # print STDERR "exec: $stdout\n"; + + # read pid from file, and draft exit script ################## + chomp ($pid=`tail -n 1 $pidfile`); + open (OUT, ">exitjob$pid.sh"); + + my $scriptheader="\#\!/bin/bash\n\#\$ -S /bin/sh\n# Both lines are needed to invoke base\n#the above line is ignored by qsub, unless parameter \"-b yes\" is set!\n\n"; + $scriptheader .="uname -a\n\n"; + $scriptheader .="cd $___WORKING_DIR\n\n"; + + print OUT $scriptheader; + + print OUT "if $qsubwrapper_exit -stdout=$stdout -stderr=$stderr -jidfile=$jidfile -pidfile=$pidfile > exitjob$pid.out 2> exitjob$pid.err ; then + echo 'succeeded' +else + echo failed with exit status \$\? + die=1 +fi +"; + print OUT "\n\n"; + + close (OUT); + # setting permissions of the script + chmod(oct(755),"exitjob$pid.sh"); + ############################################################## + + # log in submit host ######################################### + chomp(my $my_username = `whoami`); + my $ssh = Net::OpenSSH::Compat::Perl->new($submithost, debug=>0); + + print STDERR "submithost = $submithost\n"; + print STDERR "my username = $my_username\n"; + print STDERR "qusbwrapper at = $qsubwrapper\n"; + + $ssh->login(); + ############################################################## + + + if ($argvlen==9) { + if (defined $prevjid && $prevjid!=-1 && $prevjidarraysize == 1){ + $hj = "-hold_jid $prevjid"; + } elsif (defined $prevjid && $prevjidarraysize > 1){ + $hj = "-hold_jid " . join(" -hold_jid ", @prevjidarray); + } + $qsubcmd="qsub $queueparameters -o /dev/null -e /dev/null $hj exitjob$pid.sh > exitjob$pid.log 2>&1"; + $ssh->cmd("cd $___WORKING_DIR && $qsubcmd"); + } elsif ($argvlen==8) { + $qsubcmd="qsub $queueparameters -o /dev/null -e /dev/null exitjob$pid.sh > exitjob$pid.log 2>&1"; + $ssh->cmd("cd $___WORKING_DIR && $qsubcmd"); + } + print STDERR "Executing $qsubcmd in $___WORKING_DIR\n"; + +} + + diff --git a/mosesdecoder/contrib/mert-sge-nosync/generic/qsub-wrapper-exit-sge-nosync.pl b/mosesdecoder/contrib/mert-sge-nosync/generic/qsub-wrapper-exit-sge-nosync.pl new file mode 100644 index 0000000000000000000000000000000000000000..b480cbf72f89886d881b6b7bffae3e4f26226627 --- /dev/null +++ b/mosesdecoder/contrib/mert-sge-nosync/generic/qsub-wrapper-exit-sge-nosync.pl @@ -0,0 +1,312 @@ +#! /usr/bin/perl + +# $Id$ +use strict; +use Net::OpenSSH::Compat::Perl; +####################### +#Default parameters +#parameters for submiiting processes through SGE +#NOTE: group name is ws06ossmt (with 2 's') and not ws06osmt (with 1 's') +my $queueparameters=""; + +# look for the correct pwdcmd +my $pwdcmd = getPwdCmd(); + +my $workingdir = `$pwdcmd`; chomp $workingdir; +# my $tmpdir="$workingdir/tmp$$"; +# my $jobscript="$workingdir/job$$.sh"; +# my $qsubout="$workingdir/out.job$$"; +# my $qsuberr="$workingdir/err.job$$"; + + +$SIG{INT} = \&kill_all_and_quit; # catch exception for CTRL-C + +my $submithost=""; +my $help=""; +my $dbg=""; +my $version=""; +my $qsubname="WR$$"; +my $cmd=""; +my $cmdout=undef; +my $cmderr=undef; +my $jid=0; +my $jidfile=undef; +my $pid=0; +my $pidfile=undef; +my $prevjid=undef; +my $parameters=""; +my $old_sge = 0; # assume grid engine < 6.0 +my $prevjidarraysize = 0; +my $force_delete = 0; +my @prevjidarray = (); + +sub init(){ + use Getopt::Long qw(:config pass_through); + GetOptions('version'=>\$version, + 'help'=>\$help, + 'debug'=>\$dbg, + 'submithost=s'=> \$submithost, + 'qsub-prefix=s'=> \$qsubname, + 'stdout=s'=> \$cmdout, + 'stderr=s'=> \$cmderr, + 'jidfile=s'=> \$jidfile, + 'pidfile=s'=> \$pidfile, # process id for previous job + 'prevjid=s'=> \$prevjid, + 'queue-parameter=s'=> \$queueparameters, + 'force-delete=i' => \$force_delete, + 'old-sge' => \$old_sge, + ) or exit(1); + $parameters="@ARGV"; + + # read $pid from file + chomp($pid=`tail -n 1 $pidfile`); + # print "PID=+$pidfile+\n"; + + if (defined $jidfile) { + chomp($jid=`tail -n 1 $jidfile`); + } + + # print STDERR "INPUT prevjid =+$prevjid+\n"; + $prevjid =~ s/^\s+|\s+$//g; + # print STDERR "TRIMMED prevjid =+$prevjid+\n"; + + @prevjidarray = split(/\s+/,$prevjid); + $prevjidarraysize = scalar(@prevjidarray); + + # print STDERR "arraysize: $prevjidarraysize\n"; + + + version() if $version; + usage() if $help; + print_parameters() if $dbg; +} + +####################### +##print version +sub version(){ +# print STDERR "version 1.0 (29-07-2006)\n"; + print STDERR "version 1.1 (31-07-2006)\n"; + exit(1); +} + +#usage +sub usage(){ + print STDERR "qsub-wrapper.pl [options]\n"; + print STDERR "Options:\n"; + print STDERR "-stdout file to find stdout from target cmd (optional)\n"; + print STDERR "-stderr file to find stderr from target cmd (optional)\n"; + print STDERR "-jidfile file to find the submit jobid (for submit option)\n"; + print STDERR "-pidfile file to find the process id to the target job for deletion\n"; + print STDERR "-prevjid wait for the previous job with jobid=id to finish before starting (optional)\n"; + print STDERR "-force-delete 1 force-delete without checking\n"; + print STDERR "-qsub-prefix name for sumbitted jobs (optional)\n"; + print STDERR "-queue-parameters parameter for the queue (optional)\n"; + print STDERR "-old-sge ... assume Sun Grid Engine < 6.0\n"; + print STDERR "-debug debug\n"; + print STDERR "-version print version of the script\n"; + print STDERR "-help this help\n"; + exit(1); +} + +#printparameters +sub print_parameters(){ + # print STDERR "command: $cmd\n"; + if (defined($cmdout)){ print STDERR "file for stdout: $cmdout\n"; } + else { print STDERR "file for stdout is not defined, stdout is discarded\n"; } + if (defined($cmderr)){ print STDERR "file for stdout: $cmderr\n"; } + else { print STDERR "file for stderr is not defined, stderr is discarded\n"; } + if (defined($jidfile)){ print STDERR "file for submit job id: $jidfile\n"; } + else { print STDERR "file for submit job id is not defined, jidfile is discarded\n"; } + print STDERR "Qsub name: $qsubname\n"; + print STDERR "Queue parameters: $queueparameters\n"; + print STDERR "parameters directly passed to cmd: $parameters\n"; + exit(1); +} + + +####################### +#Script starts here + +init(); + +my $tmpdir="$workingdir/tmp$pid"; +my $jobscript="$workingdir/job$pid.sh"; +my $qsubout="$workingdir/out.job$pid"; +my $qsuberr="$workingdir/err.job$pid"; + + + +### usage() if $cmd eq ""; +### +### safesystem("mkdir -p $tmpdir") or die; +### +### preparing_script(); +### +#### my $maysync = $old_sge ? "" : "-sync y"; +#### never run in syn mode +###my $maysync = ""; +### +###my $qsubcmd = ""; +#### create the qsubcmd to submit to the queue with the parameter "-b yes" +####my $qsubcmd="qsub $queueparameters $maysync -V -o $qsubout -e $qsuberr -N $qsubname -b yes $jobscript > $jobscript.log 2>&1"; +### +#### add -b yes if not yet defined, otherwise leave empty +###$queueparameters .= " -b yes " if (index($queueparameters," -b ")==-1); +### +### +###if (defined $prevjid && $prevjid!=-1 && $prevjidarraysize == 1) { +### $qsubcmd="qsub $queueparameters $maysync -V -hold_jid $prevjid -o $qsubout -e $qsuberr -N $qsubname $jobscript > $jobscript.log 2>&1"; +###} elsif (defined $prevjid && $prevjidarraysize > 1) { +### my $hj = "-hold_jid " . join(" -hold_jid ", @prevjidarray); +### # print STDERR "hj is $hj\n"; +### $qsubcmd="qsub $queueparameters $maysync -V $hj -o $qsubout -e $qsuberr -N $qsubname $jobscript > $jobscript.log 2>&1"; +###} else { +### $qsubcmd="qsub $queueparameters $maysync -V -o $qsubout -e $qsuberr -N $qsubname $jobscript > $jobscript.log 2>&1"; +###} +### +###print "submitting $qsubcmd\n"; +### +####run the qsubcmd +### +###safesystem($qsubcmd) or die; +### +####getting id of submitted job############# +###my $res; +###open (IN,"$jobscript.log") or die "Can't read main job id: $jobscript.log"; +###chomp($res=); +###my @arrayStr = split(/\s+/,$res); +###my $id=$arrayStr[2]; +###die "Failed to get job id from $jobscript.log, got: $res" +### if $id !~ /^[0-9]+$/; +###close(IN); +############################################ +###print STDERR " res:$res\n"; +###print STDERR " id:$id\n"; +### +###open (JIDOUT,">$jidfile") or die "Can't open jid file to write"; +###print JIDOUT "$id\n"; +###close(JIDOUT); +### +###open (JOBNUMOUT,">$jidfile.job") or die "Can't open id.job file to write"; +###print JOBNUMOUT "$$\n"; +###close(JOBNUMOUT); +### +###if ($old_sge) { +### # need to workaround -sync, add another job that will wait for the main one +### # prepare a fake waiting script +### my $syncscript = "$jobscript.sync_workaround_script.sh"; +### safesystem("echo 'date' > $syncscript") or die; +### +### my $checkpointfile = "$jobscript.sync_workaround_checkpoint"; +### +### # ensure checkpoint does not exist +### safesystem("\\rm -f $checkpointfile") or die; +### +### # start the 'hold' job, i.e. the job that will wait +#### $cmd="qsub -cwd $queueparameters -hold_jid $id -o $checkpointfile f -e /dev/null -N $qsubname.W $syncscript >& $qsubname.W.log"; +### $cmd="qsub -cwd $queueparameters -hold_jid $id -o $checkpointfile -e /dev/null -N $qsubname.W $syncscript >& $qsubname.W.log"; +### safesystem($cmd) or die; +### +### # and wait for checkpoint file to appear +### my $nr=0; +### while (!-e $checkpointfile) { +### sleep(10); +### $nr++; +### print STDERR "w" if $nr % 3 == 0; +### } +### safesystem("\\rm -f $checkpointfile $syncscript") or die(); +### print STDERR "End of waiting workaround.\n"; +###} + + +my $failure=0; + + +if (!$force_delete) { + $failure=&check_exit_status(); + print STDERR "check_exit_status returned $failure\n"; +} + + +&kill_all_and_quit() if $failure; + +&remove_temporary_files() if !$dbg; + + + + +sub check_exit_status(){ + my $failure=0; + + print STDERR "check_exit_status of submitted job $jid from file $qsubout\n"; + open(IN,"$qsubout") or die "Can't read $qsubout"; + while (){ + $failure=1 if (/failed with exit status/); + } + close(IN); + return $failure; +} + +sub kill_all_and_quit(){ + my $my_username = undef; + + # chomp($my_username = `whoami`); + # + # my $ssh = Net::OpenSSH::Compat::Perl->new($submithost, debug=>0); + # + # $ssh->login("$my_username",`cat /home/$my_username/accpw`); + + my $ssh = Net::OpenSSH::Compat::Perl->new($submithost, debug=>0); + + $ssh->login(); + + print STDERR "kill_all_and_quit\n"; + print STDERR "qdel $jid\n"; + # safesystem("qdel $jid"); + $ssh->cmd("qdel $jid"); + + print STDERR "The submitted jobs died not correctly\n"; + print STDERR "Send qdel signal to the submitted jobs\n"; + + exit(1); +} + +sub remove_temporary_files(){ + #removing temporary files + + unlink("${jobscript}"); + unlink("${jobscript}.log"); + unlink("$qsubout"); + unlink("$qsuberr"); + rmdir("$tmpdir"); +} + +sub safesystem { + print STDERR "Executing: @_\n"; + system(@_); + if ($? == -1) { + print STDERR "Failed to execute: @_\n $!\n"; + exit(1); + } + elsif ($? & 127) { + printf STDERR "Execution of: @_\n died with signal %d, %s coredump\n", + ($? & 127), ($? & 128) ? 'with' : 'without'; + exit(1); + } + else { + my $exitcode = $? >> 8; + print STDERR "Exit code: $exitcode\n" if $exitcode; + return ! $exitcode; + } +} + +# look for the correct pwdcmd (pwd by default, pawd if it exists) +# I assume that pwd always exists +sub getPwdCmd(){ + my $pwdcmd="pwd"; + my $a; + chomp($a=`which pawd | head -1 | awk '{print $1}'`); + if ($a && -e $a){ $pwdcmd=$a; } + return $pwdcmd; +} + diff --git a/mosesdecoder/contrib/mert-sge-nosync/generic/qsub-wrapper-sge-nosync.pl b/mosesdecoder/contrib/mert-sge-nosync/generic/qsub-wrapper-sge-nosync.pl new file mode 100644 index 0000000000000000000000000000000000000000..1a17b83fe14d17c81f1c55a7af5abb07af4370fd --- /dev/null +++ b/mosesdecoder/contrib/mert-sge-nosync/generic/qsub-wrapper-sge-nosync.pl @@ -0,0 +1,320 @@ +#! /usr/bin/perl + +# $Id$ +use strict; + +####################### +#Default parameters +#parameters for submiiting processes through SGE +#NOTE: group name is ws06ossmt (with 2 's') and not ws06osmt (with 1 's') +my $queueparameters=""; + +# look for the correct pwdcmd +my $pwdcmd = getPwdCmd(); + +my $workingdir = `$pwdcmd`; chomp $workingdir; + +my $uniqtime = `date +"%s%N"`; chomp $uniqtime; +my $uid = "$$".".".$uniqtime; + + +my $tmpdir="$workingdir/tmp$uid"; +my $jobscript="$workingdir/job$uid.sh"; +my $qsubout="$workingdir/out.job$uid"; +my $qsuberr="$workingdir/err.job$uid"; + + +$SIG{INT} = \&kill_all_and_quit; # catch exception for CTRL-C + +my $help=""; +my $dbg=""; +my $version=""; +my $qsubname="WR$uid"; +my $cmd=""; +my $cmdout=undef; +my $cmderr=undef; +my $jidfile=undef; +my $pidfile=undef; +my $prevjid=undef; +my $parameters=""; +my $old_sge = 0; # assume grid engine < 6.0 +my $prevjidarraysize = 0; +my @prevjidarray = (); + +sub init(){ + use Getopt::Long qw(:config pass_through); + GetOptions('version'=>\$version, + 'help'=>\$help, + 'debug'=>\$dbg, + 'qsub-prefix=s'=> \$qsubname, + 'command=s'=> \$cmd, + 'stdout=s'=> \$cmdout, + 'stderr=s'=> \$cmderr, + 'jidfile=s'=> \$jidfile, + 'prevjid=s'=> \$prevjid, + 'queue-parameters=s'=> \$queueparameters, + 'old-sge' => \$old_sge, + ) or exit(1); + $parameters="@ARGV"; + + # print STDERR "INPUT prevjid =+$prevjid+\n"; + $prevjid =~ s/^\s+|\s+$//g; + # print STDERR "TRIMMED prevjid =+$prevjid+\n"; + + @prevjidarray = split(/\s+/,$prevjid); + $prevjidarraysize = scalar(@prevjidarray); + + # print STDERR "arraysize: $prevjidarraysize\n"; + + + + + version() if $version; + usage() if $help; + print_parameters() if $dbg; +} + +####################### +##print version +sub version(){ +# print STDERR "version 1.0 (29-07-2006)\n"; + print STDERR "version 1.1 (31-07-2006)\n"; + exit(1); +} + +#usage +sub usage(){ + print STDERR "qsub-wrapper.pl [options]\n"; + print STDERR "Options:\n"; + print STDERR "-command command to run\n"; + print STDERR "-stdout file to save stdout of cmd (optional)\n"; + print STDERR "-stderr file to save stderr of cmd (optional)\n"; + print STDERR "-jidfile file to save the submit jobid (for submit option)\n"; + print STDERR "-prevjid wait for the previous job with jobid=id to finish before starting (optional)\n"; + print STDERR "-qsub-prefix name for sumbitted jobs (optional)\n"; + print STDERR "-queue-parameters parameter for the queue (optional)\n"; + print STDERR "-old-sge ... assume Sun Grid Engine < 6.0\n"; + print STDERR "-debug debug\n"; + print STDERR "-version print version of the script\n"; + print STDERR "-help this help\n"; + exit(1); +} + +#printparameters +sub print_parameters(){ + print STDERR "command: $cmd\n"; + if (defined($cmdout)){ print STDERR "file for stdout: $cmdout\n"; } + else { print STDERR "file for stdout is not defined, stdout is discarded\n"; } + if (defined($cmderr)){ print STDERR "file for stdout: $cmderr\n"; } + else { print STDERR "file for stderr is not defined, stderr is discarded\n"; } + if (defined($jidfile)){ print STDERR "file for submit job id: $jidfile\n"; } + else { print STDERR "file for submit job id is not defined, jidfile is discarded\n"; } + print STDERR "Qsub name: $qsubname\n"; + print STDERR "Queue parameters: $queueparameters\n"; + print STDERR "parameters directly passed to cmd: $parameters\n"; + exit(1); +} + +#script creation +sub preparing_script(){ + my $scriptheader="\#\!/bin/bash\n# the above line is ignored by qsub, unless parameter \"-b yes\" is set!\n\n"; + $scriptheader.="uname -a\n\n"; + + $scriptheader.="cd $workingdir\n\n"; + + open (OUT, "> $jobscript"); + print OUT $scriptheader; + + print OUT "if $cmd $parameters > $tmpdir/cmdout$uid 2> $tmpdir/cmderr$uid ; then + echo 'succeeded' +else + echo failed with exit status \$\? + die=1 +fi +"; + + if (defined $cmdout){ + print OUT "mv -f $tmpdir/cmdout$uid $cmdout || echo failed to preserve the log: $tmpdir/cmdout$uid\n\n"; + } + else{ + print OUT "rm -f $tmpdir/cmdout$uid\n\n"; + } + + if (defined $cmderr){ + print OUT "mv -f $tmpdir/cmderr$uid $cmderr || echo failed to preserve the log: $tmpdir/cmderr$uid\n\n"; + } + else{ + print OUT "rm -f $tmpdir/cmderr$uid\n\n"; + } + print OUT "if [ x\$die == 1 ]; then exit 1; fi\n"; + close(OUT); + + #setting permissions of the script + chmod(oct(755),$jobscript); +} + +####################### +#Script starts here + +init(); + +usage() if $cmd eq ""; + +safesystem("mkdir -p $tmpdir") or die; + +preparing_script(); + +# my $maysync = $old_sge ? "" : "-sync y"; +# never run in syn mode +my $maysync = ""; + +my $qsubcmd = ""; +# create the qsubcmd to submit to the queue with the parameter "-b yes" +#my $qsubcmd="qsub $queueparameters $maysync -V -o $qsubout -e $qsuberr -N $qsubname -b yes $jobscript > $jobscript.log 2>&1"; + +# add -b yes if not yet defined, otherwise leave empty +$queueparameters .= " -b yes " if (index($queueparameters," -b ")==-1); + + +if (defined $prevjid && $prevjid!=-1 && $prevjidarraysize == 1) { + $qsubcmd="qsub $queueparameters $maysync -V -hold_jid $prevjid -o $qsubout -e $qsuberr -N $qsubname $jobscript > $jobscript.log 2>&1"; +} elsif (defined $prevjid && $prevjidarraysize > 1) { + my $hj = "-hold_jid " . join(" -hold_jid ", @prevjidarray); + # print STDERR "hj is $hj\n"; + $qsubcmd="qsub $queueparameters $maysync -V $hj -o $qsubout -e $qsuberr -N $qsubname $jobscript > $jobscript.log 2>&1"; +} else { + $qsubcmd="qsub $queueparameters $maysync -V -o $qsubout -e $qsuberr -N $qsubname $jobscript > $jobscript.log 2>&1"; +} + +print "submitting $qsubcmd\n"; + +#run the qsubcmd + +safesystem($qsubcmd) or die; + +#getting id of submitted job############# +my $res; +open (IN,"$jobscript.log") or die "Can't read main job id: $jobscript.log"; +chomp($res=); +my @arrayStr = split(/\s+/,$res); +my $id=$arrayStr[2]; +die "Failed to get job id from $jobscript.log, got: $res" + if $id !~ /^[0-9]+$/; +close(IN); +######################################### +print STDERR " res:$res\n"; +print STDERR " id:$id\n"; + +open (JIDOUT,">$jidfile") or die "Can't open jid file to write"; +print JIDOUT "$id\n"; +close(JIDOUT); + +open (PIDOUT,">$jidfile.pid") or die "Can't open id.pid file to write"; +print PIDOUT "$uid\n"; +close(PIDOUT); + +if ($old_sge) { + # need to workaround -sync, add another job that will wait for the main one + # prepare a fake waiting script + my $syncscript = "$jobscript.sync_workaround_script.sh"; + safesystem("echo 'date' > $syncscript") or die; + + my $checkpointfile = "$jobscript.sync_workaround_checkpoint"; + + # ensure checkpoint does not exist + safesystem("\\rm -f $checkpointfile") or die; + + # start the 'hold' job, i.e. the job that will wait +# $cmd="qsub -cwd $queueparameters -hold_jid $id -o $checkpointfile -e /dev/null -N $qsubname.W $syncscript >& $qsubname.W.log"; + $cmd="qsub -cwd $queueparameters -hold_jid $id -o $checkpointfile -e /dev/null -N $qsubname.W $syncscript >& $qsubname.W.log"; + safesystem($cmd) or die; + + # and wait for checkpoint file to appear + my $nr=0; + while (!-e $checkpointfile) { + sleep(10); + $nr++; + print STDERR "w" if $nr % 3 == 0; + } + safesystem("\\rm -f $checkpointfile $syncscript") or die(); + print STDERR "End of waiting workaround.\n"; +} + + + + +# my $failure=&check_exit_status(); +# print STDERR "check_exit_status returned $failure\n"; + +# &kill_all_and_quit() if $failure; + +# &remove_temporary_files() if !$dbg; + + + + + + + +sub check_exit_status(){ + my $failure=0; + + print STDERR "check_exit_status of submitted job $id\n"; + open(IN,"$qsubout") or die "Can't read $qsubout"; + while (){ + $failure=1 if (/failed with exit status/); + } + close(IN); + return $failure; +} + +sub kill_all_and_quit(){ + print STDERR "kill_all_and_quit\n"; + print STDERR "qdel $id\n"; + safesystem("qdel $id"); + + print STDERR "The submitted jobs died not correctly\n"; + print STDERR "Send qdel signal to the submitted jobs\n"; + + exit(1); +} + +sub remove_temporary_files(){ + #removing temporary files + + unlink("${jobscript}"); + unlink("${jobscript}.log"); + unlink("$qsubout"); + unlink("$qsuberr"); + rmdir("$tmpdir"); +} + +sub safesystem { + print STDERR "Executing: @_\n"; + system(@_); + if ($? == -1) { + print STDERR "Failed to execute: @_\n $!\n"; + exit(1); + } + elsif ($? & 127) { + printf STDERR "Execution of: @_\n died with signal %d, %s coredump\n", + ($? & 127), ($? & 128) ? 'with' : 'without'; + exit(1); + } + else { + my $exitcode = $? >> 8; + print STDERR "Exit code: $exitcode\n" if $exitcode; + return ! $exitcode; + } +} + +# look for the correct pwdcmd (pwd by default, pawd if it exists) +# I assume that pwd always exists +sub getPwdCmd(){ + my $pwdcmd="pwd"; + my $a; + chomp($a=`which pawd | head -1 | awk '{print $1}'`); + if ($a && -e $a){ $pwdcmd=$a; } + return $pwdcmd; +} + diff --git a/mosesdecoder/contrib/mert-sge-nosync/training/mert-moses-sge-nosync.pl b/mosesdecoder/contrib/mert-sge-nosync/training/mert-moses-sge-nosync.pl new file mode 100644 index 0000000000000000000000000000000000000000..6d6c083ce5faf77d9fba7770b29bdd0977f05270 --- /dev/null +++ b/mosesdecoder/contrib/mert-sge-nosync/training/mert-moses-sge-nosync.pl @@ -0,0 +1,1633 @@ +#! /usr/bin/perl -w +# $Id$ +# Usage: +# mert-moses.pl +# For other options see below or run 'mert-moses.pl --help' + +# Notes: +# and should be raw text files, one sentence per line +# can be a prefix, in which case the files are 0, 1, etc. are used + +# Excerpts from revision history + +# Sept 2011 multi-threaded mert (Barry Haddow) +# 3 Aug 2011 Added random directions, historic best, pairwise ranked (PK) +# Jul 2011 simplifications (Ondrej Bojar) +# -- rely on moses' -show-weights instead of parsing moses.ini +# ... so moses is also run once *before* mert starts, checking +# the model to some extent +# -- got rid of the 'triples' mess; +# use --range to supply bounds for random starting values: +# --range tm:-3..3 --range lm:-3..3 +# 5 Aug 2009 Handling with different reference length policies (shortest, average, closest) for BLEU +# and case-sensistive/insensitive evaluation (Nicola Bertoldi) +# 5 Jun 2008 Forked previous version to support new mert implementation. +# 13 Feb 2007 Better handling of default values for lambda, now works with multiple +# models and lexicalized reordering +# 11 Oct 2006 Handle different input types through parameter --inputype=[0|1] +# (0 for text, 1 for confusion network, default is 0) (Nicola Bertoldi) +# 10 Oct 2006 Allow skip of filtering of phrase tables (--no-filter-phrase-table) +# useful if binary phrase tables are used (Nicola Bertoldi) +# 28 Aug 2006 Use either closest or average or shortest (default) reference +# length as effective reference length +# Use either normalization or not (default) of texts (Nicola Bertoldi) +# 31 Jul 2006 move gzip run*.out to avoid failure wit restartings +# adding default paths +# 29 Jul 2006 run-filter, score-nbest and mert run on the queue (Nicola; Ondrej had to type it in again) +# 28 Jul 2006 attempt at foolproof usage, strong checking of input validity, merged the parallel and nonparallel version (Ondrej Bojar) +# 27 Jul 2006 adding the safesystem() function to handle with process failure +# 22 Jul 2006 fixed a bug about handling relative path of configuration file (Nicola Bertoldi) +# 21 Jul 2006 adapted for Moses-in-parallel (Nicola Bertoldi) +# 18 Jul 2006 adapted for Moses and cleaned up (PK) +# 21 Jan 2005 unified various versions, thorough cleanup (DWC) +# now indexing accumulated n-best list solely by feature vectors +# 14 Dec 2004 reimplemented find_threshold_points in C (NMD) +# 25 Oct 2004 Use either average or shortest (default) reference +# length as effective reference length (DWC) +# 13 Oct 2004 Use alternative decoders (DWC) +# Original version by Philipp Koehn + +use strict; +use Net::OpenSSH::Compat::Perl; +use FindBin qw($Bin); +use File::Basename; +use File::Path; +use File::Spec; +use Cwd; + +my $SCRIPTS_ROOTDIR = $Bin; +$SCRIPTS_ROOTDIR =~ s/\/training$//; +$SCRIPTS_ROOTDIR = $ENV{"SCRIPTS_ROOTDIR"} if defined($ENV{"SCRIPTS_ROOTDIR"}); + +## We preserve this bit of comments to keep the traditional weight ranges. +# "w" => [ [ 0.0, -1.0, 1.0 ] ], # word penalty +# "d" => [ [ 1.0, 0.0, 2.0 ] ], # lexicalized reordering model +# "lm" => [ [ 1.0, 0.0, 2.0 ] ], # language model +# "g" => [ [ 1.0, 0.0, 2.0 ], # generation model +# [ 1.0, 0.0, 2.0 ] ], +# "tm" => [ [ 0.3, 0.0, 0.5 ], # translation model +# [ 0.2, 0.0, 0.5 ], +# [ 0.3, 0.0, 0.5 ], +# [ 0.2, 0.0, 0.5 ], +# [ 0.0,-1.0, 1.0 ] ], # ... last weight is phrase penalty +# "lex"=> [ [ 0.1, 0.0, 0.2 ] ], # global lexical model +# "I" => [ [ 0.0,-1.0, 1.0 ] ], # input lattice scores + + + +# moses.ini file uses FULL names for lambdas, while this training script +# internally (and on the command line) uses ABBR names. +my @ABBR_FULL_MAP = qw(d=weight-d lm=weight-l tm=weight-t w=weight-w + g=weight-generation lex=weight-lex I=weight-i); +my %ABBR2FULL = map {split/=/,$_,2} @ABBR_FULL_MAP; +my %FULL2ABBR = map {my ($a, $b) = split/=/,$_,2; ($b, $a);} @ABBR_FULL_MAP; + +my $minimum_required_change_in_weights = 0.00001; + # stop if no lambda changes more than this + +my $verbose = 0; +my $usage = 0; # request for --help + +# We assume that if you don't specify working directory, +# we set the default is set to `pwd`/mert-work + +my $___FIRSTJOBWAITID = undef; # wait for a job in the grid before first job starts + +my $___WORKING_DIR = File::Spec->catfile(Cwd::getcwd(), "mert-work"); +my $___DEV_F = undef; # required, input text to decode +my $___DEV_E = undef; # required, basename of files with references +my $___DECODER = undef; # required, pathname to the decoder executable +my $___CONFIG = undef; # required, pathname to startup ini file +my $___N_BEST_LIST_SIZE = 100; +my $___LATTICE_SAMPLES = 0; +my $queue_flags = "-hard"; # extra parameters for parallelizer + # the -l ws0ssmt was relevant only to JHU 2006 workshop +my $___JOBS = undef; # if parallel, number of jobs to use (undef or 0 -> serial) +my $___DECODER_FLAGS = ""; # additional parametrs to pass to the decoder +my $qsubprefix = ""; +my $continue = 0; # should we try to continue from the last saved step? +my $skip_decoder = 0; # and should we skip the first decoder run (assuming we got interrupted during mert) +my $___FILTER_PHRASE_TABLE = 1; # filter phrase table +my $___PREDICTABLE_SEEDS = 0; +my $___START_WITH_HISTORIC_BESTS = 0; # use best settings from all previous iterations as starting points [Foster&Kuhn,2009] +my $___RANDOM_DIRECTIONS = 0; # search in random directions only +my $___NUM_RANDOM_DIRECTIONS = 0; # number of random directions, also works with default optimizer [Cer&al.,2008] +my $___PAIRWISE_RANKED_OPTIMIZER = 0; # use Hopkins&May[2011] +my $___PRO_STARTING_POINT = 0; # get a starting point from pairwise ranked optimizer +my $___RANDOM_RESTARTS = 20; +my $___HISTORIC_INTERPOLATION = 0; # interpolate optimize weights with previous iteration's weights [Hopkins&May,2011,5.4.3] +my $__THREADS = 0; + +# Parameter for effective reference length when computing BLEU score +# Default is to use shortest reference +# Use "--shortest" to use shortest reference length +# Use "--average" to use average reference length +# Use "--closest" to use closest reference length +# Only one between --shortest, --average and --closest can be set +# If more than one choice the defualt (--shortest) is used +my $___SHORTEST = 0; +my $___AVERAGE = 0; +my $___CLOSEST = 0; + +# Use "--nocase" to compute case-insensitive scores +my $___NOCASE = 0; + +# Use "--nonorm" to non normalize translation before computing scores +my $___NONORM = 0; + +# set 0 if input type is text, set 1 if input type is confusion network +my $___INPUTTYPE = 0; + + +my $mertdir = undef; # path to new mert directory +my $mertargs = undef; # args to pass through to mert & extractor +my $mertmertargs = undef; # args to pass through to mert only +my $extractorargs = undef; # args to pass through to extractor only +my $filtercmd = undef; # path to filter-model-given-input.pl +my $submithost = undef; +my $filterfile = undef; +my $qsubwrapper = undef; +my $qsubwrapper_exit = undef; +my $moses_parallel_cmd = undef; +my $poll_decoder_cmd = undef; +my $zipextcmd = undef; +my $zipextargs = undef; +my $processresultcmd = undef; +my $processresultargs = undef; +my $old_sge = 0; # assume sge<6.0 +my $___CONFIG_ORIG = undef; # pathname to startup ini file before filtering +my $___ACTIVATE_FEATURES = undef; # comma-separated (or blank-separated) list of features to work on + # if undef work on all features + # (others are fixed to the starting values) +my $___RANGES = undef; +my $prev_aggregate_nbl_size = -1; # number of previous step to consider when loading data (default =-1) + # -1 means all previous, i.e. from iteration 1 + # 0 means no previous data, i.e. from actual iteration + # 1 means 1 previous data , i.e. from the actual iteration and from the previous one + # and so on +my $maximum_iterations = 25; +my $cmd = undef; + +##################### +my $processfeatlistcmd = undef; +my $processfeatlistargs = undef; +my $createconfigcmd = undef; +my $createconfigargs = undef; +my $decoderargs = undef; +##################### + +use Getopt::Long; +GetOptions( + "prevjid=i" => \$___FIRSTJOBWAITID, + "working-dir=s" => \$___WORKING_DIR, + "input=s" => \$___DEV_F, + "inputtype=i" => \$___INPUTTYPE, + "refs=s" => \$___DEV_E, + "decoder=s" => \$___DECODER, + "config=s" => \$___CONFIG, + "nbest=i" => \$___N_BEST_LIST_SIZE, + "lattice-samples=i" => \$___LATTICE_SAMPLES, + "submithost=s" => \$submithost, + "queue-flags=s" => \$queue_flags, + "jobs=i" => \$___JOBS, + "decoder-flags=s" => \$___DECODER_FLAGS, + "continue" => \$continue, + "skip-decoder" => \$skip_decoder, + "shortest" => \$___SHORTEST, + "average" => \$___AVERAGE, + "closest" => \$___CLOSEST, + "nocase" => \$___NOCASE, + "nonorm" => \$___NONORM, + "help" => \$usage, + "verbose" => \$verbose, + "mertdir=s" => \$mertdir, + "mertargs=s" => \$mertargs, + "extractorargs=s" => \$extractorargs, + "mertmertargs=s" => \$mertmertargs, + "rootdir=s" => \$SCRIPTS_ROOTDIR, + "filtercmd=s" => \$filtercmd, # allow to override the default location + "filterfile=s" => \$filterfile, # input to filtering script (useful for lattices/confnets) + "qsubwrapper=s" => \$qsubwrapper, # allow to override the default location + "mosesparallelcmd=s" => \$moses_parallel_cmd, # allow to override the default location + "old-sge" => \$old_sge, #passed to moses-parallel + "filter-phrase-table!" => \$___FILTER_PHRASE_TABLE, # (dis)allow of phrase tables + "predictable-seeds" => \$___PREDICTABLE_SEEDS, # make random restarts deterministic + "historic-bests" => \$___START_WITH_HISTORIC_BESTS, # use best settings from all previous iterations as starting points + "random-directions" => \$___RANDOM_DIRECTIONS, # search only in random directions + "number-of-random-directions=i" => \$___NUM_RANDOM_DIRECTIONS, # number of random directions + "random-restarts=i" => \$___RANDOM_RESTARTS, # number of random restarts + "activate-features=s" => \$___ACTIVATE_FEATURES, #comma-separated (or blank-separated) list of features to work on (others are fixed to the starting values) + "range=s@" => \$___RANGES, + "prev-aggregate-nbestlist=i" => \$prev_aggregate_nbl_size, #number of previous step to consider when loading data (default =-1, i.e. all previous) + "maximum-iterations=i" => \$maximum_iterations, + "pairwise-ranked" => \$___PAIRWISE_RANKED_OPTIMIZER, + "pro-starting-point" => \$___PRO_STARTING_POINT, + "historic-interpolation=f" => \$___HISTORIC_INTERPOLATION, + "threads=i" => \$__THREADS +) or exit(1); + +# the 4 required parameters can be supplied on the command line directly +# or using the --options +if (scalar @ARGV == 4) { + # required parameters: input_file references_basename decoder_executable + $___DEV_F = shift; + $___DEV_E = shift; + $___DECODER = shift; + $___CONFIG = shift; +} + +if ($usage || !defined $___DEV_F || !defined $___DEV_E || !defined $___DECODER || !defined $___CONFIG) { + print STDERR "usage: $0 input-text references decoder-executable decoder.ini +Options: + --prevjid=i ... previous job SGE ID to wait before first job starts + --working-dir=mert-dir ... where all the files are created + --nbest=100 ... how big nbestlist to generate + --lattice-samples ... how many lattice samples (Chatterjee & Cancedda, emnlp 2010) + --jobs=N ... set this to anything to run moses in parallel + --mosesparallelcmd=STR ... use a different script instead of moses-parallel + --submithost=STRING ... submithost from where qsub operates + --queue-flags=STRING ... anything you with to pass to qsub, eg. + '-l ws06osssmt=true'. The default is: '-hard' + To reset the parameters, please use + --queue-flags=' ' + (i.e. a space between the quotes). + --decoder-flags=STRING ... extra parameters for the decoder + --continue ... continue from the last successful iteration + --skip-decoder ... skip the decoder run for the first time, + assuming that we got interrupted during + optimization + --shortest --average --closest + ... Use shortest/average/closest reference length + as effective reference length (mutually exclusive) + --nocase ... Do not preserve case information; i.e. + case-insensitive evaluation (default is false). + --nonorm ... Do not use text normalization (flag is not active, + i.e. text is NOT normalized) + --filtercmd=STRING ... path to filter-model-given-input.pl + --filterfile=STRING ... path to alternative to input-text for filtering + model. useful for lattice decoding + --rootdir=STRING ... where do helpers reside (if not given explicitly) + --mertdir=STRING ... path to new mert implementation + --mertargs=STRING ... extra args for both extractor and mert + --extractorargs=STRING ... extra args for extractor only + --mertmertargs=STRING ... extra args for mert only + --scorenbestcmd=STRING ... path to score-nbest.py + --old-sge ... passed to parallelizers, assume Grid Engine < 6.0 + --inputtype=[0|1|2] ... Handle different input types: (0 for text, + 1 for confusion network, 2 for lattices, + default is 0) + --no-filter-phrase-table ... disallow filtering of phrase tables + (useful if binary phrase tables are available) + --random-restarts=INT ... number of random restarts (default: 20) + --predictable-seeds ... provide predictable seeds to mert so that random + restarts are the same on every run + --range=tm:0..1,-1..1 ... specify min and max value for some features + --range can be repeated as needed. + The order of the various --range specifications + is important only within a feature name. + E.g.: + --range=tm:0..1,-1..1 --range=tm:0..2 + is identical to: + --range=tm:0..1,-1..1,0..2 + but not to: + --range=tm:0..2 --range=tm:0..1,-1..1 + --activate-features=STRING ... comma-separated list of features to optimize, + others are fixed to the starting values + default: optimize all features + example: tm_0,tm_4,d_0 + --prev-aggregate-nbestlist=INT ... number of previous step to consider when + loading data (default = $prev_aggregate_nbl_size) + -1 means all previous, i.e. from iteration 1 + 0 means no previous data, i.e. only the + current iteration + N means this and N previous iterations + + --maximum-iterations=ITERS ... Maximum number of iterations. Default: $maximum_iterations + --random-directions ... search only in random directions + --number-of-random-directions=int ... number of random directions + (also works with regular optimizer, default: 0) + --pairwise-ranked ... Use PRO for optimisation (Hopkins and May, emnlp 2011) + --pro-starting-point ... Use PRO to get a starting point for MERT + --threads=NUMBER ... Use multi-threaded mert (must be compiled in). + --historic-interpolation ... Interpolate optimized weights with prior iterations' weight + (parameter sets factor [0;1] given to current weights) +"; + exit 1; +} + + +# Check validity of input parameters and set defaults if needed + +print STDERR "Using SCRIPTS_ROOTDIR: $SCRIPTS_ROOTDIR\n"; + +# path of script for filtering phrase tables and running the decoder +$filtercmd="$SCRIPTS_ROOTDIR/training/filter-model-given-input.pl" if !defined $filtercmd; + +if ( ! -x $filtercmd && ! $___FILTER_PHRASE_TABLE) { + print STDERR "Filtering command not found: $filtercmd.\n"; + print STDERR "Use --filtercmd=PATH to specify a valid one or --no-filter-phrase-table\n"; + exit 1; +} + +$qsubwrapper = "$SCRIPTS_ROOTDIR/generic/qsub-wrapper-sge-nosync.pl" if !defined $qsubwrapper; + +$qsubwrapper_exit = "$SCRIPTS_ROOTDIR/generic/qsub-wrapper-exit-sge-nosync.pl" if !defined $qsubwrapper_exit; + +$moses_parallel_cmd = "$SCRIPTS_ROOTDIR/generic/moses-parallel-sge-nosync.pl" + if !defined $moses_parallel_cmd; + +if (!defined $mertdir) { + $mertdir = "$SCRIPTS_ROOTDIR/../mert"; + print STDERR "Assuming --mertdir=$mertdir\n"; +} + +my $mert_extract_cmd = "$mertdir/extractor"; +my $mert_mert_cmd = "$mertdir/mert"; +my $mert_pro_cmd = "$mertdir/pro"; + +die "Not executable: $mert_extract_cmd" if ! -x $mert_extract_cmd; +die "Not executable: $mert_mert_cmd" if ! -x $mert_mert_cmd; +die "Not executable: $mert_pro_cmd" if ! -x $mert_pro_cmd; + +my $pro_optimizer = "$mertdir/megam_i686.opt"; # or set to your installation +if (($___PAIRWISE_RANKED_OPTIMIZER || $___PRO_STARTING_POINT) && ! -x $pro_optimizer) { + print "did not find $pro_optimizer, installing it in $mertdir\n"; + `cd $mertdir; wget http://www.cs.utah.edu/~hal/megam/megam_i686.opt.gz;`; + `gunzip $pro_optimizer.gz`; + `chmod +x $pro_optimizer`; + die("ERROR: Installation of megam_i686.opt failed! Install by hand from http://www.cs.utah.edu/~hal/megam/") unless -x $pro_optimizer; +} + +$mertargs = "" if !defined $mertargs; + +my $scconfig = undef; +if ($mertargs =~ /\-\-scconfig\s+(.+?)(\s|$)/){ + $scconfig=$1; + $scconfig =~ s/\,/ /g; + $mertargs =~ s/\-\-scconfig\s+(.+?)(\s|$)//; +} + +# handling reference lengh strategy +if (($___CLOSEST + $___AVERAGE + $___SHORTEST) > 1){ + die "You can specify just ONE reference length strategy (closest or shortest or average) not both\n"; +} + +if ($___SHORTEST){ + $scconfig .= " reflen:shortest"; +}elsif ($___AVERAGE){ + $scconfig .= " reflen:average"; +}elsif ($___CLOSEST){ + $scconfig .= " reflen:closest"; +} + +# handling case-insensitive flag +if ($___NOCASE) { + $scconfig .= " case:false"; +}else{ + $scconfig .= " case:true"; +} +$scconfig =~ s/^\s+//; +$scconfig =~ s/\s+$//; +$scconfig =~ s/\s+/,/g; + +$scconfig = "--scconfig $scconfig" if ($scconfig); + +my $mert_extract_args=$mertargs; +$mert_extract_args .=" $scconfig"; +$mert_extract_args .=" $extractorargs"; + +$mertmertargs = "" if !defined $mertmertargs; + +my $mert_mert_args="$mertargs $mertmertargs"; +$mert_mert_args =~ s/\-+(binary|b)\b//; +$mert_mert_args .=" $scconfig"; +if ($___ACTIVATE_FEATURES){ $mert_mert_args .=" -o \"$___ACTIVATE_FEATURES\""; } + +my ($just_cmd_filtercmd,$x) = split(/ /,$filtercmd); +die "Not executable: $just_cmd_filtercmd" if ! -x $just_cmd_filtercmd; +die "Not executable: $moses_parallel_cmd" if defined $___JOBS && ! -x $moses_parallel_cmd; +die "Not executable: $qsubwrapper" if defined $___JOBS && ! -x $qsubwrapper; +die "Not executable: $___DECODER" if ! -x $___DECODER; + +my $input_abs = ensure_full_path($___DEV_F); +die "File not found: $___DEV_F (interpreted as $input_abs)." + if ! -e $input_abs; +$___DEV_F = $input_abs; + +# Option to pass to qsubwrapper and moses-parallel +my $pass_old_sge = $old_sge ? "-old-sge" : ""; + +my $decoder_abs = ensure_full_path($___DECODER); +die "File not executable: $___DECODER (interpreted as $decoder_abs)." + if ! -x $decoder_abs; +$___DECODER = $decoder_abs; + +my $ref_abs = ensure_full_path($___DEV_E); +# check if English dev set (reference translations) exist and store a list of all references +my @references; +if (-e $ref_abs) { + push @references, $ref_abs; +} +else { + # if multiple file, get a full list of the files + my $part = 0; + if (! -e $ref_abs."0" && -e $ref_abs.".ref0") { + $ref_abs .= ".ref"; + } + while (-e $ref_abs.$part) { + push @references, $ref_abs.$part; + $part++; + } + die("Reference translations not found: $___DEV_E (interpreted as $ref_abs)") unless $part; +} + +my $config_abs = ensure_full_path($___CONFIG); +die "File not found: $___CONFIG (interpreted as $config_abs)." + if ! -e $config_abs; +$___CONFIG = $config_abs; + +# moses should use our config +if ($___DECODER_FLAGS =~ /(^|\s)-(config|f) / +|| $___DECODER_FLAGS =~ /(^|\s)-(ttable-file|t) / +|| $___DECODER_FLAGS =~ /(^|\s)-(distortion-file) / +|| $___DECODER_FLAGS =~ /(^|\s)-(generation-file) / +|| $___DECODER_FLAGS =~ /(^|\s)-(lmodel-file) / +|| $___DECODER_FLAGS =~ /(^|\s)-(global-lexical-file) / +) { + die "It is forbidden to supply any of -config, -ttable-file, -distortion-file, -generation-file or -lmodel-file in the --decoder-flags.\nPlease use only the --config option to give the config file that lists all the supplementary files."; +} + +# as weights are normalized in the next steps (by cmert) +# normalize initial LAMBDAs, too +my $need_to_normalize = 1; + +#store current directory and create the working directory (if needed) +my $cwd = `pawd 2>/dev/null`; +if(!$cwd){$cwd = `pwd`;} +chomp($cwd); + +mkpath($___WORKING_DIR); + +{ +# open local scope + +#chdir to the working directory +chdir($___WORKING_DIR) or die "Can't chdir to $___WORKING_DIR"; + +# fixed file names +my $mert_outfile = "mert.out"; +my $mert_logfile = "mert.log"; +my $weights_in_file = "init.opt"; +my $weights_out_file = "weights.txt"; + +# set start run +my $start_run = 1; ## START FROM run>1 is not supported +my $bestpoint = undef; +my $devbleu = undef; +my $sparse_weights_file = undef; +my $jobid = -1; + +my $prev_feature_file = undef; +my $prev_score_file = undef; +my $prev_init_file = undef; + + +######################### +# set jobid to trace different jobs +my $prevjid = undef; + + +################################################## +# STEP 1: FILTER PHRASE TABLE #################### +################################################## + +if ($___FILTER_PHRASE_TABLE) { + my $outdir = "filtered"; + if (-e "$outdir/moses.ini") { + print STDERR "Assuming the tables are already filtered, reusing $outdir/moses.ini\n"; + } + else { + # filter the phrase tables with respect to input, use --decoder-flags + print STDERR "filtering the phrase tables... ".`date`; + my $___FILTER_F = $___DEV_F; + $___FILTER_F = $filterfile if (defined $filterfile); + $cmd = "$filtercmd ./$outdir $___CONFIG $___FILTER_F"; + if (!defined $___FIRSTJOBWAITID) { + # &submit_or_exec($cmd,"filterphrases.out","filterphrases.err","filterphrases.id"); + $qsubprefix = "filph"; + &submit_or_exec_thu_host($submithost,"","",$cmd,"","filterphrases.out","filterphrases.err","filterphrases.id"); + } else { + # &submit_or_exec($cmd,"filterphrases.out","filterphrases.err","filterphrases.id",$___FIRSTJOBWAITID); + $qsubprefix = "filph"; + &submit_or_exec_thu_host($submithost,"","",$cmd,"","filterphrases.out","filterphrases.err","filterphrases.id",$___FIRSTJOBWAITID); + } + chomp($jobid=`tail -n 1 filterphrases.id`); + $prevjid = $jobid; + print STDERR "JOBID for filterphrases is $prevjid\n"; + ## clear up tmp + &exit_submit_thu_host($submithost,"","","","filterphrases.out","filterphrases.err","filterphrases.id","filterphrases.id.pid",$prevjid); + } + + # make a backup copy of startup ini filepath + $___CONFIG_ORIG = $___CONFIG; + # the decoder should now use the filtered model + $___CONFIG = "$outdir/moses.ini"; +} +else{ + # do not filter phrase tables (useful if binary phrase tables are available) + # use the original configuration file + $___CONFIG_ORIG = $___CONFIG; +} + +################################################# +######## STEP 2: CHECK moses.ini ################ +################################################# + + +# path of script for filtering phrase tables and running the decoder ------------ +$processfeatlistcmd="$SCRIPTS_ROOTDIR/training/sge-nosync/process-featlist-sge-nosync.pl" if !defined $processfeatlistcmd; + +$processfeatlistargs = "" if !defined $processfeatlistargs; +$processfeatlistargs = "$processfeatlistargs --range $___RANGES" if (defined $___RANGES); +$processfeatlistargs = "$processfeatlistargs --decoder-flags $___DECODER_FLAGS" if (!$___DECODER_FLAGS eq ""); + +$cmd = "$processfeatlistcmd $___DECODER $___CONFIG --inputtype $___INPUTTYPE $processfeatlistargs"; + +if (defined $prevjid) { + $qsubprefix = "proclist"; + &submit_or_exec_thu_host($submithost,"","",$cmd,"","processfeatlist.out","processfeatlist.err","processfeatlist.id",$prevjid); +} else { + $qsubprefix = "proclist"; + &submit_or_exec_thu_host($submithost,"","",$cmd,"","processfeatlist.out","processfeatlist.err","processfeatlist.id"); +} +chomp($jobid=`tail -n 1 processfeatlist.id`); +$prevjid = $jobid; +#---------------------------------------------------------------------------------- +## clear up tmp +&exit_submit_thu_host($submithost,"","","","processfeatlist.out","processfeatlist.err","processfeatlist.id","processfeatlist.id.pid",$prevjid); + +# we run moses to check validity of moses.ini and to obtain all the feature +# names +##COPIED# my $featlist = get_featlist_from_moses($___CONFIG); +##COPIED#$featlist = insert_ranges_to_featlist($featlist, $___RANGES); +##COPIED# +##COPIED## Mark which features are disabled: +##COPIED#if (defined $___ACTIVATE_FEATURES) { +##COPIED# my %enabled = map { ($_, 1) } split /[, ]+/, $___ACTIVATE_FEATURES; +##COPIED# my %cnt; +##COPIED# for(my $i=0; $i{"names"}}); $i++) { +##COPIED# my $name = $featlist->{"names"}->[$i]; +##COPIED# $cnt{$name} = 0 if !defined $cnt{$name}; +##COPIED# $featlist->{"enabled"}->[$i] = $enabled{$name."_".$cnt{$name}}; +##COPIED# $cnt{$name}++; +##COPIED# } +##COPIED#} else { +##COPIED# # all enabled +##COPIED# for(my $i=0; $i{"names"}}); $i++) { +##COPIED# $featlist->{"enabled"}->[$i] = 1; +##COPIED# } +##COPIED#} +##COPIED# +##COPIED#print STDERR "MERT starting values and ranges for random generation:\n"; +##COPIED#for(my $i=0; $i{"names"}}); $i++) { +##COPIED# my $name = $featlist->{"names"}->[$i]; +##COPIED# my $val = $featlist->{"values"}->[$i]; +##COPIED# my $min = $featlist->{"mins"}->[$i]; +##COPIED# my $max = $featlist->{"maxs"}->[$i]; +##COPIED# my $enabled = $featlist->{"enabled"}->[$i]; +##COPIED# printf STDERR " %5s = %7.3f", $name, $val; +##COPIED# if ($enabled) { +##COPIED# printf STDERR " (%5.2f .. %5.2f)\n", $min, $max; +##COPIED# } else { +##COPIED# print STDERR " --- inactive, not optimized ---\n"; +##COPIED# } +##COPIED#} + +############################################################## +# if continue from last section ###### +###################################### +##!#if ($continue) { +##!# # getting the last finished step +##!# print STDERR "Trying to continue an interrupted optimization.\n"; +##!# open IN, "finished_step.txt" or die "Failed to find the step number, failed to read finished_step.txt"; +##!# my $step = ; +##!# chomp $step; +##!# close IN; +##!# +##!# print STDERR "Last finished step is $step\n"; +##!# +##!# # getting the first needed step +##!# my $firststep; +##!# if ($prev_aggregate_nbl_size==-1){ +##!# $firststep=1; +##!# } +##!# else{ +##!# $firststep=$step-$prev_aggregate_nbl_size+1; +##!# $firststep=($firststep>0)?$firststep:1; +##!# } +##!# +##!##checking if all needed data are available +##!## get $prev_feature_file, $prev_score_file, $prev_init_file +##!# if ($firststep<=$step){ +##!# print STDERR "First previous needed data index is $firststep\n"; +##!# print STDERR "Checking whether all needed data (from step $firststep to step $step) are available\n"; +##!# +##!# for (my $prevstep=$firststep; $prevstep<=$step;$prevstep++){ +##!# print STDERR "Checking whether data of step $prevstep are available\n"; +##!# if (! -e "run$prevstep.features.dat"){ +##!# die "Can't start from step $step, because run$prevstep.features.dat was not found!"; +##!# }else{ +##!# if (defined $prev_feature_file){ +##!# $prev_feature_file = "${prev_feature_file},run$prevstep.features.dat"; +##!# } +##!# else{ +##!# $prev_feature_file = "run$prevstep.features.dat"; +##!# } +##!# } +##!# if (! -e "run$prevstep.scores.dat"){ +##!# die "Can't start from step $step, because run$prevstep.scores.dat was not found!"; +##!# }else{ +##!# if (defined $prev_score_file){ +##!# $prev_score_file = "${prev_score_file},run$prevstep.scores.dat"; +##!# } +##!# else{ +##!# $prev_score_file = "run$prevstep.scores.dat"; +##!# } +##!# } +##!# if (! -e "run$prevstep.${weights_in_file}"){ +##!# die "Can't start from step $step, because run$prevstep.${weights_in_file} was not found!"; +##!# }else{ +##!# if (defined $prev_init_file){ +##!# $prev_init_file = "${prev_init_file},run$prevstep.${weights_in_file}"; +##!# } +##!# else{ +##!# $prev_init_file = "run$prevstep.${weights_in_file}"; +##!# } +##!# } +##!# } +##!# if (! -e "run$step.weights.txt"){ +##!# die "Can't start from step $step, because run$step.weights.txt was not found!"; +##!# } +##!# if (! -e "run$step.$mert_logfile"){ +##!# die "Can't start from step $step, because run$step.$mert_logfile was not found!"; +##!# } +##!# if (! -e "run$step.best$___N_BEST_LIST_SIZE.out.gz"){ +##!# die "Can't start from step $step, because run$step.best$___N_BEST_LIST_SIZE.out.gz was not found!"; +##!# } +##!# print STDERR "All needed data are available\n"; +##!# +##!# print STDERR "Loading information from last step ($step)\n"; +##!# my %dummy; # sparse features +##!# ($bestpoint,$devbleu) = &get_weights_from_mert("run$step.$mert_outfile","run$step.$mert_logfile",scalar @{$featlist->{"names"}},\%dummy); +##!# die "Failed to parse mert.log, missed Best point there." +##!# if !defined $bestpoint || !defined $devbleu; +##!# print "($step) BEST at $step $bestpoint => $devbleu at ".`date`; +##!# my @newweights = split /\s+/, $bestpoint; +##!# +##!# # Sanity check: order of lambdas must match +##!# sanity_check_order_of_lambdas($featlist, +##!# "gunzip -c < run$step.best$___N_BEST_LIST_SIZE.out.gz |"); +##!# +##!# # update my cache of lambda values +##!# $featlist->{"values"} = \@newweights; +##!# } +##!# else{ +##!# print STDERR "No previous data are needed\n"; +##!# } +##!# +##!# $start_run = $step +1; +##!# } +##################################### +## If continue from last section #### +##################################### + + +print STDERR "I am about to start main loop!!!\n"; + +### load featlist when needed +# my $featlist = get_featlist_from_moses($___CONFIG); +# my $featlist = undef; +###### MERT MAIN LOOP + + + +my $run=$start_run-1; + +my $oldallsorted = undef; +my $allsorted = undef; + +my $nbest_file=undef; +my $lsamp_file=undef; #Lattice samples +my $orig_nbest_file=undef; # replaced if lattice sampling + + + +# ------------------------------------------------------------------------ +# ------ get_featlist_and_create_config (only for run 1) ----------------- +# ------------------------------------------------------------------------ +$createconfigcmd="$SCRIPTS_ROOTDIR/training/sge-nosync/create-config-sge-nosync.pl" if !defined $createconfigcmd; + +$createconfigargs = "" if !defined $createconfigargs; +$createconfigargs = "$createconfigargs --range $___RANGES" if (defined $___RANGES); +$createconfigargs = "$createconfigargs --decoder-flags $___DECODER_FLAGS" if (!$___DECODER_FLAGS eq ""); +$createconfigargs = "$createconfigargs --devbleu $devbleu" if (defined $devbleu); +$createconfigargs = "$createconfigargs --sparse_weights_file $sparse_weights_file" if (defined $sparse_weights_file); +$createconfigargs = "$createconfigargs --working-dir $___WORKING_DIR" if (defined $___WORKING_DIR); + +my $cmd = "$createconfigcmd $___DEV_F $___DECODER $___CONFIG first --inputtype $___INPUTTYPE $createconfigargs"; + +if (defined $prevjid) { + $qsubprefix = "firstcfg"; + &submit_or_exec_thu_host($submithost,"","",$cmd,"","createconfigfirstrun.out","createconfigfirstrun.err","createconfigfirstrun.id",$prevjid); +} else { + $qsubprefix = "firstcfg"; + &submit_or_exec_thu_host($submithost,"","",$cmd,"","createconfigfirstrun.out","createconfigfirstrun.err","createconfigfirstrun.id"); +} +chomp($jobid=`tail -n 1 createconfigfirstrun.id`); +$prevjid = $jobid; +## clear up tmp +&exit_submit_thu_host($submithost,"","","","createconfigfirstrun.out","createconfigfirstrun.err","createconfigfirstrun.id","createconfigfirstrun.id.pid",$prevjid); + +##COPIED# create_config($___CONFIG, "./run$run.moses.ini", $featlist, $run, (defined$devbleu?$devbleu:"--not-estimated--"),$sparse_weights_file); +# ------------------------------------------------------------------------- +# ------------------------------------------------------------------------- +print "create config for first run is done!\n"; + + +while(1) { + $run++; + if ($maximum_iterations && $run > $maximum_iterations) { + print "Maximum number of iterations exceeded - stopping\n"; + last; + } + # run beamdecoder with option to output nbestlists + # the end result should be (1) @NBEST_LIST, a list of lists; (2) @SCORE, a list of lists of lists + + print "run $run start at ".`date`; + + + + + ################################################### + # step 3: decode + ################################################### + +# path of script for running the decoder ------------ +$moses_parallel_cmd="$SCRIPTS_ROOTDIR/generic/moses-parallel-sge-nosync.pl" if !defined $moses_parallel_cmd; +my $decoder_cmd; + +if (defined $___JOBS && $___JOBS > 0 ) { + # not support -lattice-samples + # $decoder_cmd = "$moses_parallel_cmd $pass_old_sge -config $___CONFIG -inputtype $___INPUTTYPE -queue-parameters \"$queue_flags\" -decoder-flags $___DECODER_FLAGS -lattice-samples $___LATTICE_SAMPLES -n-best-list-size $___N_BEST_LIST_SIZE -input-file $___DEV_F -jobs $___JOBS -decoder $___DECODER -run $run -need-to-normalize $need_to_normalize -working-dir $___WORKING_DIR -qsubwrapper $qsubwrapper > run$run.out"; + + $decoderargs = ""; + ### DO NOT pass $___DECODER_FLAGS!! load the latest decoding parameter from inside the script + # $decoderargs = "-decoder-flags $___DECODER_FLAGS " if (!$___DECODER_FLAGS eq ""); + $decoderargs = "-script-rootdir $SCRIPTS_ROOTDIR " if (!$SCRIPTS_ROOTDIR eq ""); + $decoderargs = "$decoderargs -lattice-samples $___LATTICE_SAMPLES " if ($___LATTICE_SAMPLES != 0); + + $decoder_cmd = "$moses_parallel_cmd $pass_old_sge -config $___CONFIG -inputtype $___INPUTTYPE -submithost \"$submithost\" -queue-parameters \"$queue_flags\" $decoderargs -n-best-list-size $___N_BEST_LIST_SIZE -input-file $___DEV_F -jobs $___JOBS -decoder $___DECODER -run $run -need-to-normalize $need_to_normalize -working-dir $___WORKING_DIR -qsubwrapper $qsubwrapper -qsubwrapper-exit $qsubwrapper_exit > run$run.out"; +} else { + print STDERR "Execute without going through grid is not supported!\n"; + exit(1); +} + +if (defined $prevjid) { + $qsubprefix="decode$run"; + &submit_or_exec_thu_host($submithost,"","",$decoder_cmd,"","decode$run.out","decode$run.err","decode$run.id",$prevjid); +} else { + $qsubprefix="decode$run"; + &submit_or_exec_thu_host($submithost,"","",$decoder_cmd,"","decode$run.out","decode$run.err","decode$run.id"); +} +## may have to change to decode$run.last.id to retrieve later job id. +$need_to_normalize = 0; +chomp($jobid=`tail -n 1 decode$run.id`); +$prevjid = $jobid; + +# safesystem($decode_cmd) or die "The decoder died. CONFIG WAS $decoder_config \n"; + + ################################################### + # step 4: define wait job + ################################################### + $poll_decoder_cmd="$SCRIPTS_ROOTDIR/training/sge-nosync/poll-decoder.pl -poll-target decode$run.W.out"; + + if (defined $prevjid) { + $qsubprefix = "poll$run"; + &submit_or_exec_thu_host($submithost,"","",$poll_decoder_cmd,"","decode$run.POLL.out","decode$run.POLL.err","decode$run.POLL.id",$prevjid); + } else { + die "Step 4 (poll_decoder): Cannot find previous process for sequential submit\n"; + } + + chomp($jobid=`tail -n 1 decode$run.POLL.id`); + $prevjid = $jobid; + + ################################################## + # clear-up (step 3 and step 4) + ################################################## + # job for moses-parallel-sge-nosync.pl + &exit_submit_thu_host($submithost,"","","","decode$run.out","decode$run.err","decode$run.id","decode$run.id.pid",$prevjid); + ## waitall.sh job (cannot submit here because pid not established) + ##&exit_submit("decode$run.W.out","decode$run.W.err","decode$run.W.id","decode$run.W.id.pid",$prevjid); + # polling job + &exit_submit_thu_host($submithost,"","","","decode$run.POLL.out","decode$run.POLL.err","decode$run.POLL.id","decode$run.POLL.id.pid",$prevjid); + + + + ################################################### + # step 5: zip and extract + ################################################### + $zipextcmd="$SCRIPTS_ROOTDIR/training/sge-nosync/zipextract-decoder-result.pl" if !defined $zipextcmd; + $zipextargs=" -run $run"; + $zipextargs.=" -mertdir $mertdir" if defined $mertdir; + $zipextargs.=" $___DEV_E ./run$run.moses.ini"; + + $cmd = "$zipextcmd $zipextargs"; + + if (defined $prevjid){ + $qsubprefix="zip$run"; + &submit_or_exec_thu_host($submithost,"","",$cmd,"","zipext$run.out","zipext$run.err","zipext$run.id",$prevjid); + } else { + die "Step 5 (zip extract): Cannot find previous process for sequential submit\n"; + } + chomp($jobid=`tail -n 1 zipext$run.id`); + $prevjid = $jobid; + + ## clear-up tmp ### + &exit_submit_thu_host($submithost,"","","","zipext$run.out","zipext$run.err","zipext$run.id","zipext$run.id.pid",$prevjid); + + #################################################### + # step 6: process results (mert) + #################################################### + $processresultcmd="$SCRIPTS_ROOTDIR/training/sge-nosync/process-moses-result-sge-nosync.pl" if !defined $processresultcmd; + $processresultargs=" -run $run -submithost $submithost -queue-flags \"$queue_flags\""; + $processresultargs.=" -inputtype $___INPUTTYPE"; + $processresultargs.=" -mertdir $mertdir" if defined $mertdir; +# $processresultargs.=" $___DEV_F $___DECODER ./run$run.moses.ini"; + $processresultargs.=" $___DEV_F $___DECODER $___CONFIG"; + + $cmd = "$processresultcmd $processresultargs"; + + if (defined $prevjid){ + $qsubprefix="MERT$run"; + &submit_or_exec_thu_host($submithost,"","",$cmd,"","processmoses$run.out","processmoses$run.err","processmoses$run.id",$prevjid); + } else { + die "Step 6 (process results - (mert)): Cannot find previous process for sequential submit\n"; + } + chomp($jobid=`tail -n 1 processmoses$run.id`); + $prevjid = $jobid; + + ## clear-up tmp ## + &exit_submit_thu_host($submithost,"","","","processmoses$run.out","processmoses$run.err","processmoses$run.id","processmoses$run.id.pid",$prevjid); + + ### sleep to prevent batch select in short time + sleep(1); + +##!# else { +##!# $nbest_file="run$run.best$___N_BEST_LIST_SIZE.out.gz"; +##!# print "skipped decoder run $run\n"; +##!# $skip_decoder = 0; +##!# $need_to_normalize = 0; +##!# } +} + + +################################################# +# step 7: write out the final files +################################################# +$createconfigcmd="$SCRIPTS_ROOTDIR/training/sge-nosync/create-config-sge-nosync.pl" if !defined $createconfigcmd; + +$createconfigargs = "" if !defined $createconfigargs; +$createconfigargs = "$createconfigargs --range $___RANGES" if (defined $___RANGES); +$createconfigargs = "$createconfigargs --decoder-flags $___DECODER_FLAGS" if (!$___DECODER_FLAGS eq ""); +$createconfigargs = "$createconfigargs --devbleu $devbleu" if (defined $devbleu); +$createconfigargs = "$createconfigargs --sparse_weights_file $sparse_weights_file" if (defined $sparse_weights_file); +$createconfigargs = "$createconfigargs --working-dir $___WORKING_DIR" if (defined $___WORKING_DIR); + +my $cmd = "$createconfigcmd $___DEV_F $___DECODER $___CONFIG_ORIG final --inputtype $___INPUTTYPE $createconfigargs"; + +if (defined $prevjid) { + $qsubprefix = "finalcfg"; + &submit_or_exec_thu_host($submithost,"","",$cmd,"","createconfigfinal.out","createconfigfinal.err","createconfigfinal.id",$prevjid); +} else { + $qsubprefix = "finalcfg"; + &submit_or_exec_thu_host($submithost,"","",$cmd,"","createconfigfinal.out","createconfigfinal.err","createconfigfinal.id"); +} +chomp($jobid=`tail -n 1 createconfigfinal.id`); +$prevjid = $jobid; +# clear up tmp +&exit_submit_thu_host($submithost,"","","","createconfigfinal.out","createconfigfinal.err","createconfigfinal.id","createconfigfinal.id.pid",$prevjid); +## ------------------------------------------------------------------------- +print "create config for last run is done!\n"; + + +###### STEP 8: clear job ###### +my $clearcmd = ""; +$clearcmd = "$SCRIPTS_ROOTDIR/training/sge-nosync/cleartmpfiles.pl"; + + +if (defined $prevjid){ + $qsubprefix="Clearjob"; + &submit_or_exec_thu_host($submithost,"","",$clearcmd,"","clear.out","clear.err","clear.id",$prevjid); +} else { + die "Step 7 (clear job): Cannot find previous process for sequential submit\n"; +} +chomp($jobid=`tail -n 1 clear.id`); +$prevjid = $jobid; + +## clear-up tmp ## +&exit_submit_thu_host($submithost,"","","","clear.out","clear.err","clear.id","clear.id.pid",$prevjid); + +print "Tuning finished at ".`date`; + +chdir($cwd); + +} # end of local scope + +sub get_weights_from_mert { + my ($outfile,$logfile,$weight_count,$sparse_weights) = @_; + my ($bestpoint,$devbleu); + if ($___PAIRWISE_RANKED_OPTIMIZER || ($___PRO_STARTING_POINT && $logfile =~ /pro/)) { + open(IN,$outfile) or die "Can't open $outfile"; + my (@WEIGHT,$sum); + for(my $i=0;$i<$weight_count;$i++) { push @WEIGHT, 0; } + while() { + # regular features + if (/^F(\d+) ([\-\.\de]+)/) { + $WEIGHT[$1] = $2; + $sum += abs($2); + } + # sparse features + elsif(/^(.+_.+) ([\-\.\de]+)/) { + $$sparse_weights{$1} = $2; + } + } + $devbleu = "unknown"; + foreach (@WEIGHT) { $_ /= $sum; } + foreach (keys %{$sparse_weights}) { $$sparse_weights{$_} /= $sum; } + $bestpoint = join(" ",@WEIGHT); + close IN; + } + else { + open(IN,$logfile) or die "Can't open $logfile"; + while () { + if (/Best point:\s*([\s\d\.\-e]+?)\s*=> ([\-\d\.]+)/) { + $bestpoint = $1; + $devbleu = $2; + last; + } + } + close IN; + } + return ($bestpoint,$devbleu); +} + +##COPIED# sub run_decoder { +##COPIED# my ($featlist, $run, $need_to_normalize) = @_; +##COPIED# my $filename_template = "run%d.best$___N_BEST_LIST_SIZE.out"; +##COPIED# my $filename = sprintf($filename_template, $run); +##COPIED# my $lsamp_filename = undef; +##COPIED# if ($___LATTICE_SAMPLES) { +##COPIED# my $lsamp_filename_template = "run%d.lsamp$___LATTICE_SAMPLES.out"; +##COPIED# $lsamp_filename = sprintf($lsamp_filename_template, $run); +##COPIED# } +##COPIED# +##COPIED# # user-supplied parameters +##COPIED# print "params = $___DECODER_FLAGS\n"; +##COPIED# +##COPIED# # parameters to set all model weights (to override moses.ini) +##COPIED# my @vals = @{$featlist->{"values"}}; +##COPIED# if ($need_to_normalize) { +##COPIED# print STDERR "Normalizing lambdas: @vals\n"; +##COPIED# my $totlambda=0; +##COPIED# grep($totlambda+=abs($_),@vals); +##COPIED# grep($_/=$totlambda,@vals); +##COPIED# } +##COPIED# # moses now does not seem accept "-tm X -tm Y" but needs "-tm X Y" +##COPIED# my %model_weights; +##COPIED# for(my $i=0; $i{"names"}}); $i++) { +##COPIED# my $name = $featlist->{"names"}->[$i]; +##COPIED# $model_weights{$name} = "-$name" if !defined $model_weights{$name}; +##COPIED# $model_weights{$name} .= sprintf " %.6f", $vals[$i]; +##COPIED# } +##COPIED# my $decoder_config = join(" ", values %model_weights); +##COPIED# $decoder_config .= " -weight-file run$run.sparse-weights" if -e "run$run.sparse-weights"; +##COPIED# print STDERR "DECODER_CFG = $decoder_config\n"; +##COPIED# print "decoder_config = $decoder_config\n"; +##COPIED# +##COPIED# +##COPIED# # run the decoder +##COPIED# my $decoder_cmd; +##COPIED# my $lsamp_cmd = ""; +##COPIED# if ($___LATTICE_SAMPLES) { +##COPIED# $lsamp_cmd = " -lattice-samples $lsamp_filename $___LATTICE_SAMPLES "; +##COPIED# } +##COPIED# +##COPIED# ####### RUN moses_parallel ################################# +##COPIED# # need to gather: +##COPIED# ### $run +##COPIED# ### $lsamp_cmd +##COPIED# ### $filename +##COPIED# ########################################################### +##COPIED# +##COPIED# if (defined $___JOBS && $___JOBS > 0) { +##COPIED# $decoder_cmd = "$moses_parallel_cmd $pass_old_sge -config $___CONFIG -inputtype $___INPUTTYPE -qsub-prefix mert$run -queue-parameters \"$queue_flags\" -decoder-parameters \"$___DECODER_FLAGS $decoder_config\" $lsamp_cmd -n-best-list \"$filename $___N_BEST_LIST_SIZE\" -input-file $___DEV_F -jobs $___JOBS -decoder $___DECODER > run$run.out"; +##COPIED# } else { +##COPIED# $decoder_cmd = "$___DECODER $___DECODER_FLAGS -config $___CONFIG -inputtype $___INPUTTYPE $decoder_config $lsamp_cmd -n-best-list $filename $___N_BEST_LIST_SIZE -input-file $___DEV_F > run$run.out"; +##COPIED# } +##COPIED# +##COPIED# safesystem($decoder_cmd) or die "The decoder died. CONFIG WAS $decoder_config \n"; +##COPIED# +##COPIED# sanity_check_order_of_lambdas($featlist, $filename); +##COPIED# return ($filename, $lsamp_filename); +##COPIED# } + + +##COPIED# sub insert_ranges_to_featlist { +##COPIED# my $featlist = shift; +##COPIED# my $ranges = shift; +##COPIED# +##COPIED# $ranges = [] if !defined $ranges; +##COPIED# +##COPIED# # first collect the ranges from options +##COPIED# my $niceranges; +##COPIED# foreach my $range (@$ranges) { +##COPIED# my $name = undef; +##COPIED# foreach my $namedpair (split /,/, $range) { +##COPIED# if ($namedpair =~ /^(.*?):/) { +##COPIED# $name = $1; +##COPIED# $namedpair =~ s/^.*?://; +##COPIED# die "Unrecognized name '$name' in --range=$range" +##COPIED# if !defined $ABBR2FULL{$name}; +##COPIED# } +##COPIED# my ($min, $max) = split /\.\./, $namedpair; +##COPIED# die "Bad min '$min' in --range=$range" if $min !~ /^-?[0-9.]+$/; +##COPIED# die "Bad max '$max' in --range=$range" if $min !~ /^-?[0-9.]+$/; +##COPIED# die "No name given in --range=$range" if !defined $name; +##COPIED# push @{$niceranges->{$name}}, [$min, $max]; +##COPIED# } +##COPIED# } +##COPIED# +##COPIED# # now populate featlist +##COPIED# my $seen = undef; +##COPIED# for(my $i=0; $i{"names"}}); $i++) { +##COPIED# my $name = $featlist->{"names"}->[$i]; +##COPIED# $seen->{$name} ++; +##COPIED# my $min = 0.0; +##COPIED# my $max = 1.0; +##COPIED# if (defined $niceranges->{$name}) { +##COPIED# my $minmax = shift @{$niceranges->{$name}}; +##COPIED# ($min, $max) = @$minmax if defined $minmax; +##COPIED# } +##COPIED# $featlist->{"mins"}->[$i] = $min; +##COPIED# $featlist->{"maxs"}->[$i] = $max; +##COPIED# } +##COPIED# return $featlist; +##COPIED# } + +##COPIED# sub sanity_check_order_of_lambdas { +##COPIED# my $featlist = shift; +##COPIED# my $filename_or_stream = shift; +##COPIED# +##COPIED# my @expected_lambdas = @{$featlist->{"names"}}; +##COPIED# my @got = get_order_of_scores_from_nbestlist($filename_or_stream); +##COPIED# die "Mismatched lambdas. Decoder returned @got, we expected @expected_lambdas" +##COPIED# if "@got" ne "@expected_lambdas"; +##COPIED# } + +#### get_featlist_from_moses(): also used in process-featlist-sge-nosync.pl ##### +########################################################################### +sub get_featlist_from_moses { + # run moses with the given config file and return the list of features and + # their initial values + my $configfn = shift; + my $featlistfn = "./features.list"; + if (-e $featlistfn) { + print STDERR "Using cached features list: $featlistfn\n"; + } else { + print STDERR "Asking moses for feature names and values from $___CONFIG\n"; + my $cmd = "$___DECODER $___DECODER_FLAGS -config $configfn -inputtype $___INPUTTYPE -show-weights > $featlistfn"; + safesystem($cmd) or die "Failed to run moses with the config $configfn"; + } + + # read feature list + my @names = (); + my @startvalues = (); + open(INI,$featlistfn) or die "Can't read $featlistfn"; + my $nr = 0; + my @errs = (); + while () { + $nr++; + chomp; + /^(.+) (\S+) (\S+)$/ || die("invalid feature: $_"); + my ($longname, $feature, $value) = ($1,$2,$3); + next if $value eq "sparse"; + push @errs, "$featlistfn:$nr:Bad initial value of $feature: $value\n" + if $value !~ /^[+-]?[0-9.e]+$/; + push @errs, "$featlistfn:$nr:Unknown feature '$feature', please add it to \@ABBR_FULL_MAP\n" + if !defined $ABBR2FULL{$feature}; + push @names, $feature; + push @startvalues, $value; + } + close INI; + if (scalar @errs) { + print STDERR join("", @errs); + exit 1; + } + return {"names"=>\@names, "values"=>\@startvalues}; +} +#### get_featlist_from_moses() ends ##################################### + +sub get_order_of_scores_from_nbestlist { + # read the first line and interpret the ||| label: num num num label2: num ||| column in nbestlist + # return the score labels in order + my $fname_or_source = shift; + # print STDERR "Peeking at the beginning of nbestlist to get order of scores: $fname_or_source\n"; + open IN, $fname_or_source or die "Failed to get order of scores from nbestlist '$fname_or_source'"; + my $line = ; + close IN; + die "Line empty in nbestlist '$fname_or_source'" if !defined $line; + my ($sent, $hypo, $scores, $total) = split /\|\|\|/, $line; + $scores =~ s/^\s*|\s*$//g; + die "No scores in line: $line" if $scores eq ""; + + my @order = (); + my $label = undef; + my $sparse = 0; # we ignore sparse features here + foreach my $tok (split /\s+/, $scores) { + if ($tok =~ /.+_.+:/) { + $sparse = 1; + } elsif ($tok =~ /^([a-z][0-9a-z]*):/i) { + $label = $1; + } elsif ($tok =~ /^-?[-0-9.e]+$/) { + if (!$sparse) { + # a score found, remember it + die "Found a score but no label before it! Bad nbestlist '$fname_or_source'!" + if !defined $label; + push @order, $label; + } + $sparse = 0; + } else { + die "Not a label, not a score '$tok'. Failed to parse the scores string: '$scores' of nbestlist '$fname_or_source'"; + } + } + print STDERR "The decoder returns the scores in this order: @order\n"; + return @order; +} + +##COPIED#sub create_config { +##COPIED# my $infn = shift; # source config +##COPIED# my $outfn = shift; # where to save the config +##COPIED# my $featlist = shift; # the lambdas we should write +##COPIED# my $iteration = shift; # just for verbosity +##COPIED# my $bleu_achieved = shift; # just for verbosity +##COPIED# my $sparse_weights_file = shift; # only defined when optimizing sparse features +##COPIED# +##COPIED# my %P; # the hash of all parameters we wish to override +##COPIED# +##COPIED# # first convert the command line parameters to the hash +##COPIED# { # ensure local scope of vars +##COPIED# my $parameter=undef; +##COPIED# print "Parsing --decoder-flags: |$___DECODER_FLAGS|\n"; +##COPIED# $___DECODER_FLAGS =~ s/^\s*|\s*$//; +##COPIED# $___DECODER_FLAGS =~ s/\s+/ /; +##COPIED# foreach (split(/ /,$___DECODER_FLAGS)) { +##COPIED# if (/^\-([^\d].*)$/) { +##COPIED# $parameter = $1; +##COPIED# $parameter = $ABBR2FULL{$parameter} if defined($ABBR2FULL{$parameter}); +##COPIED# } +##COPIED# else { +##COPIED# die "Found value with no -paramname before it: $_" +##COPIED# if !defined $parameter; +##COPIED# push @{$P{$parameter}},$_; +##COPIED# } +##COPIED# } +##COPIED# } +##COPIED# +##COPIED# # First delete all weights params from the input, we're overwriting them. +##COPIED# # Delete both short and long-named version. +##COPIED# for(my $i=0; $i{"names"}}); $i++) { +##COPIED# my $name = $featlist->{"names"}->[$i]; +##COPIED# delete($P{$name}); +##COPIED# delete($P{$ABBR2FULL{$name}}); +##COPIED# } +##COPIED# +##COPIED# # Convert weights to elements in P +##COPIED# for(my $i=0; $i{"names"}}); $i++) { +##COPIED# my $name = $featlist->{"names"}->[$i]; +##COPIED# my $val = $featlist->{"values"}->[$i]; +##COPIED# $name = defined $ABBR2FULL{$name} ? $ABBR2FULL{$name} : $name; +##COPIED# # ensure long name +##COPIED# push @{$P{$name}}, $val; +##COPIED# } +##COPIED# +##COPIED# if (defined($sparse_weights_file)) { +##COPIED# push @{$P{"weights-file"}}, $___WORKING_DIR."/".$sparse_weights_file; +##COPIED# } +##COPIED# +##COPIED# # create new moses.ini decoder config file by cloning and overriding the original one +##COPIED# open(INI,$infn) or die "Can't read $infn"; +##COPIED# delete($P{"config"}); # never output +##COPIED# print "Saving new config to: $outfn\n"; +##COPIED# open(OUT,"> $outfn") or die "Can't write $outfn"; +##COPIED# print OUT "# MERT optimized configuration\n"; +##COPIED# print OUT "# decoder $___DECODER\n"; +##COPIED# print OUT "# BLEU $bleu_achieved on dev $___DEV_F\n"; +##COPIED# print OUT "# We were before running iteration $iteration\n"; +##COPIED# print OUT "# finished ".`date`; +##COPIED# my $line = ; +##COPIED# while(1) { +##COPIED# last unless $line; +##COPIED# +##COPIED# # skip until hit [parameter] +##COPIED# if ($line !~ /^\[(.+)\]\s*$/) { +##COPIED# $line = ; +##COPIED# print OUT $line if $line =~ /^\#/ || $line =~ /^\s+$/; +##COPIED# next; +##COPIED# } +##COPIED# +##COPIED# # parameter name +##COPIED# my $parameter = $1; +##COPIED# $parameter = $ABBR2FULL{$parameter} if defined($ABBR2FULL{$parameter}); +##COPIED# print OUT "[$parameter]\n"; +##COPIED# +##COPIED# # change parameter, if new values +##COPIED# if (defined($P{$parameter})) { +##COPIED# # write new values +##COPIED# foreach (@{$P{$parameter}}) { +##COPIED# print OUT $_."\n"; +##COPIED# } +##COPIED# delete($P{$parameter}); +##COPIED# # skip until new parameter, only write comments +##COPIED# while($line = ) { +##COPIED# print OUT $line if $line =~ /^\#/ || $line =~ /^\s+$/; +##COPIED# last if $line =~ /^\[/; +##COPIED# last unless $line; +##COPIED# } +##COPIED# next; +##COPIED# } +##COPIED# +##COPIED# # unchanged parameter, write old +##COPIED# while($line = ) { +##COPIED# last if $line =~ /^\[/; +##COPIED# print OUT $line; +##COPIED# } +##COPIED# } +##COPIED# +##COPIED# # write all additional parameters +##COPIED# foreach my $parameter (keys %P) { +##COPIED# print OUT "\n[$parameter]\n"; +##COPIED# foreach (@{$P{$parameter}}) { +##COPIED# print OUT $_."\n"; +##COPIED# } +##COPIED# } +##COPIED# +##COPIED# close(INI); +##COPIED# close(OUT); +##COPIED# print STDERR "Saved: $outfn\n"; +##COPIED#} + +sub safesystem { + print STDERR "Executing: @_\n"; + system(@_); + if ($? == -1) { + print STDERR "Failed to execute: @_\n $!\n"; + exit(1); + } + elsif ($? & 127) { + printf STDERR "Execution of: @_\n died with signal %d, %s coredump\n", + ($? & 127), ($? & 128) ? 'with' : 'without'; + exit(1); + } + else { + my $exitcode = $? >> 8; + print STDERR "Exit code: $exitcode\n" if $exitcode; + return ! $exitcode; + } +} +sub ensure_full_path { + my $PATH = shift; +$PATH =~ s/\/nfsmnt//; + return $PATH if $PATH =~ /^\//; + my $dir = `pawd 2>/dev/null`; + if(!$dir){$dir = `pwd`;} + chomp($dir); + $PATH = $dir."/".$PATH; + $PATH =~ s/[\r\n]//g; + $PATH =~ s/\/\.\//\//g; + $PATH =~ s/\/+/\//g; + my $sanity = 0; + while($PATH =~ /\/\.\.\// && $sanity++<10) { + $PATH =~ s/\/+/\//g; + $PATH =~ s/\/[^\/]+\/\.\.\//\//g; + } + $PATH =~ s/\/[^\/]+\/\.\.$//; + $PATH =~ s/\/+$//; +$PATH =~ s/\/nfsmnt//; + return $PATH; +} + +sub submit_or_exec { + + my $argvlen = @_; + my $cmd = undef; + my $stdout = undef; + my $stderr = undef; + my $jidfile = undef; + my $prevjid = undef; + + # if supply 3 arguments, exec without submit + # if supply 4 arguments, then submit new job + # if supply 5 arguments, wait for the previous job to finish + if ($argvlen == 3){ + ($cmd,$stdout,$stderr) = @_; + } elsif ($argvlen == 4){ + ($cmd,$stdout,$stderr,$jidfile) = @_; + } elsif ($argvlen == 5){ + ($cmd,$stdout,$stderr,$jidfile,$prevjid) = @_; + } + + print STDERR "exec: $cmd\n"; + if (defined $___JOBS && $___JOBS > 0 && $argvlen==5) { + safesystem("$qsubwrapper $pass_old_sge -command='$cmd' -queue-parameters=\"$queue_flags\" -stdout=$stdout -stderr=$stderr -jidfile=$jidfile -prevjid=$prevjid" ) + or die "ERROR: Failed to submit '$cmd' (via $qsubwrapper)"; + } + elsif (defined $___JOBS && $___JOBS > 0 && $argvlen==4) { + safesystem("$qsubwrapper $pass_old_sge -command='$cmd' -queue-parameters=\"$queue_flags\" -stdout=$stdout -stderr=$stderr -jidfile=$jidfile" ) + or die "ERROR: Failed to submit '$cmd' (via $qsubwrapper)"; + } else { + safesystem("$cmd > $stdout 2> $stderr") or die "ERROR: Failed to run '$cmd'."; + } +} + +sub exit_submit { + + my $argvlen = @_; + my $cmd = undef; + my $stdout = undef; + my $stderr = undef; + my $jidfile = undef; + my $pidfile = undef; + my $prevjid = undef; + my $prevjidarraysize = 0; + my @prevjidarray = (); + my $pid = undef; + my $qsubcmd=""; + my $hj=""; + + # if supply 4 arguments, then submit new job + # if supply 5 arguments, wait for the previous job to finish + if ($argvlen == 2) { + ($stdout,$stderr) = @_; + } elsif ($argvlen == 4){ + ($stdout,$stderr,$jidfile,$pidfile) = @_; + } elsif ($argvlen == 5){ + ($stdout,$stderr,$jidfile,$pidfile,$prevjid) = @_; + } + + # parse prevjid ######################## + $prevjid =~ s/^\s+|\s+$//g; + @prevjidarray = split(/\s+/,$prevjid); + $prevjidarraysize = scalar(@prevjidarray); + ######################################## + + + # print STDERR "exec: $stdout\n"; + + # read pid from file, and draft exit script ################## + chomp ($pid=`tail -n 1 $pidfile`); + open (OUT, ">exitjob$pid.sh"); + + my $scriptheader="\#\!/bin/bash\n\#\$ -S /bin/sh\n# Both lines are needed to invoke base\n#the above line is ignored by qsub, unless parameter \"-b yes\" is set!\n\n"; + $scriptheader .="uname -a\n\n"; + $scriptheader .="cd $___WORKING_DIR\n\n"; + + print OUT $scriptheader; + + print OUT "if $qsubwrapper_exit -stdout=$stdout -stderr=$stderr -jidfile=$jidfile -pidfile=$pidfile > exitjob$pid.out 2> exitjob$pid.err ; then + echo 'succeeded' +else + echo failed with exit status \$\? + die=1 +fi +"; + print OUT "\n\n"; + + close (OUT); + # setting permissions of the script + chmod(oct(755),"exitjob$pid.sh"); + ############################################################## + + + if (defined $___JOBS && $___JOBS > 0 && $argvlen==5) { + if (defined $prevjid && $prevjid!=-1 && $prevjidarraysize == 1){ + $hj = "-hold_jid $prevjid"; + } elsif (defined $prevjid && $prevjidarraysize > 1){ + $hj = "-hold_jid " . join(" -hold_jid ", @prevjidarray); + } + $qsubcmd="qsub $queue_flags -o /dev/null -e /dev/null -V $hj exitjob$pid.sh > exitjob$pid.log 2>&1"; + safesystem($qsubcmd) or die "ERROR: Failed to exit-submit $pid (via $qsubwrapper_exit)"; + } elsif (defined $___JOBS && $___JOBS > 0 && $argvlen==4) { + $qsubcmd="qsub $queue_flags -o /dev/null -e /dev/null -V exitjob$pid.sh > exitjob$pid.log 2>&1"; + safesystem($qsubcmd) or die "ERROR: Failed to exit-submit $pid (via $qsubwrapper_exit)"; + } else { + safesystem("rm $stdout") or die "ERROR: Failed to remove '$stdout'."; + safesystem("rm $stderr") or die "ERROR: Failed to remove '$stderr'."; + } +} + + + + + + + +sub create_extractor_script() +{ + my ($cmd, $outdir) = @_; + my $script_path = File::Spec->catfile($outdir, "extractor.sh"); + + open my $out, '>', $script_path + or die "Couldn't open $script_path for writing: $!\n"; + print $out "#!/bin/bash\n"; + print $out "cd $outdir\n"; + print $out "$cmd\n"; + close($out); + + `chmod +x $script_path`; + + return $script_path; +} + + +sub submit_or_exec_thu_host { + # use Net::OpenSSH::Compat::Perl; + + my $argvlen = @_; + my $submithost = undef; + my $run = -1; + my $idx = ""; + my $batch_and_join = ""; + my $my_username = undef; + my $cmd = undef; + my $qsubwrapcmd = undef; + my $stdout = undef; + my $stderr = undef; + my $jidfile = undef; + my $prevjid = undef; + + my $qsubname = undef; + + # if supply 7 arguments, exec without submit + # if supply 8 arguments, then submit new job + # if supply 9 arguments, wait for the previous job to finish + if ($argvlen == 7){ + ($submithost,$run,$idx,$cmd,$batch_and_join,$stdout,$stderr) = @_; + } elsif ($argvlen == 8){ + ($submithost,$run,$idx,$cmd,$batch_and_join,$stdout,$stderr,$jidfile) = @_; + } elsif ($argvlen == 9){ + ($submithost,$run,$idx,$cmd,$batch_and_join,$stdout,$stderr,$jidfile,$prevjid) = @_; + } + + + + # chomp($my_username = `whoami`); + # my $ssh = Net::OpenSSH::Compat::Perl->new($submithost, debug=>0); + # + # print STDERR "submithost = $submithost\n"; + # print STDERR "my username = $my_username\n"; + # print STDERR "qusbwrapper at = $qsubwrapper\n"; + # + # $ssh->login("$my_username",`cat /home/$my_username/accpw`); + my $ssh = Net::OpenSSH::Compat::Perl->new($submithost, debug=>0); + + print STDERR "submithost = $submithost\n"; + print STDERR "qusbwrapper at = $qsubwrapper\n"; + + $ssh->login(); + + #### extra for mert-moses-sge-nosync.pl #### + if ($qsubprefix eq ""){ + $qsubname = $stdout; + } else { + $qsubname = $qsubprefix; + } + my $queueparameters = $queue_flags; + ############################################ + + # $cmd="qsub $queueparameters $batch_and_join -o $qsubout$idx -e $qsuberr$idx -N $qsubname$idx ${jobscript}${idx}.bash > ${jobscript}${idx}.log 2>&1"; + # -o qsubout + # -e qsuberr + if ($argvlen == 9) { + $qsubwrapcmd = "$qsubwrapper -command='$cmd' -queue-parameter=\"$queueparameters $batch_and_join\" -qsub-prefix='$qsubname$idx' -stdout=$stdout -stderr=$stderr -jidfile=$jidfile -prevjid='$prevjid'"; + } elsif ($argvlen == 8) { + # $qsubwrapcmd = "$qsubwrapper -command='${jobscript}${idx}.bash' -queue-parameter=\"$queueparameters $batch_and_join\" -qsub-prefix='$qsubname$idx' -stdout=$stdout -stderr=$stderr -jidfile=$jidfile"; + $qsubwrapcmd = "$qsubwrapper -command='$cmd' -queue-parameter=\"$queueparameters $batch_and_join\" -qsub-prefix='$qsubname$idx' -stdout=$stdout -stderr=$stderr -jidfile=$jidfile"; + } + print STDERR "Executing $qsubwrapcmd in $___WORKING_DIR\n"; + $ssh->cmd("cd $___WORKING_DIR && $qsubwrapcmd"); + +} + +sub exit_submit_thu_host { + + my $argvlen = @_; + my $submithost = undef; + my $run = -1; + my $idx = ""; + my $batch_and_join = ""; + my $my_username = undef; + my $cmd = undef; + my $stdout = undef; + my $stderr = undef; + my $jidfile = undef; + my $pidfile = undef; + my $prevjid = undef; + my $prevjidarraysize = 0; + my @prevjidarray = (); + my $pid = undef; + my $qsubcmd=""; + my $hj=""; + + # if supply 8 arguments, then submit new job + # if supply 9 arguments, wait for the previous job to finish + if ($argvlen == 6){ + ($submithost,$run,$idx,$batch_and_join,$stdout,$stderr) = @_; + } elsif ($argvlen == 8){ + ($submithost,$run,$idx,$batch_and_join,$stdout,$stderr,$jidfile,$pidfile) = @_; + } elsif ($argvlen == 9){ + ($submithost,$run,$idx,$batch_and_join,$stdout,$stderr,$jidfile,$pidfile,$prevjid) = @_; + } + + # parse prevjid ######################## + $prevjid =~ s/^\s+|\s+$//g; + @prevjidarray = split(/\s+/,$prevjid); + $prevjidarraysize = scalar(@prevjidarray); + ######################################## + + #### extra for mert-moses-sge-nosync.pl #### + my $queueparameters = $queue_flags; + ############################################ + + + + # print STDERR "exec: $stdout\n"; + + # read pid from file, and draft exit script ################## + chomp ($pid=`tail -n 1 $pidfile`); + open (OUT, ">exitjob$pid.sh"); + + my $scriptheader="\#\!/bin/bash\n\#\$ -S /bin/sh\n# Both lines are needed to invoke base\n#the above line is ignored by qsub, unless parameter \"-b yes\" is set!\n\n"; + $scriptheader .="uname -a\n\n"; + $scriptheader .="cd $___WORKING_DIR\n\n"; + + print OUT $scriptheader; + + print OUT "if $qsubwrapper_exit -submithost=$submithost -stdout=$stdout -stderr=$stderr -jidfile=$jidfile -pidfile=$pidfile > exitjob$pid.out 2> exitjob$pid.err ; then + echo 'succeeded' +else + echo failed with exit status \$\? + die=1 +fi +"; + print OUT "\n\n"; + + close (OUT); + # setting permissions of the script + chmod(oct(755),"exitjob$pid.sh"); + ############################################################## + + # log in submit host ######################################### + # chomp($my_username = `whoami`); + # my $ssh = Net::OpenSSH::Compat::Perl->new($submithost, debug=>0); + # + # print STDERR "submithost = $submithost\n"; + # print STDERR "my username = $my_username\n"; + # print STDERR "qusbwrapper at = $qsubwrapper\n"; + # + # $ssh->login("$my_username",`cat /home/$my_username/accpw`); + ############################################################## + my $ssh = Net::OpenSSH::Compat::Perl->new($submithost, debug=>0); + + print STDERR "submithost = $submithost\n"; + print STDERR "qusbwrapper at = $qsubwrapper\n"; + + $ssh->login(); + + + if ($argvlen==9) { + if (defined $prevjid && $prevjid!=-1 && $prevjidarraysize == 1){ + $hj = "-hold_jid $prevjid"; + } elsif (defined $prevjid && $prevjidarraysize > 1){ + $hj = "-hold_jid " . join(" -hold_jid ", @prevjidarray); + } + $qsubcmd="qsub $queueparameters -o /dev/null -e /dev/null $hj exitjob$pid.sh > exitjob$pid.log 2>&1"; + $ssh->cmd("cd $___WORKING_DIR && $qsubcmd"); + } elsif ($argvlen==8) { + $qsubcmd="qsub $queueparameters -o /dev/null -e /dev/null exitjob$pid.sh > exitjob$pid.log 2>&1"; + $ssh->cmd("cd $___WORKING_DIR && $qsubcmd"); + } + print STDERR "Executing $qsubcmd in $___WORKING_DIR\n"; + +} + + diff --git a/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/cleartmpfiles.pl b/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/cleartmpfiles.pl new file mode 100644 index 0000000000000000000000000000000000000000..4e5e7d0460fba5010f3be6303f5cd69f55d8c8ec --- /dev/null +++ b/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/cleartmpfiles.pl @@ -0,0 +1,64 @@ +#!/usr/bin/perl + + + my @filename_id = ""; + my $this_id = ""; + + # remove exitjob and forceexitjob + chomp(my @rddfile_list = `ls exitjob* forceexitjob*`); + foreach my $rddfile (@rddfile_list) { + unlink("$rddfile"); + } + + chomp(@filename_id = `ls *.id | grep -v 'clear'`); + open (OUT, "> all.id.all"); + print OUT "==Combine log at ".`date`; + print OUT `tail -n +1 *.id`; + print OUT "==LOG combined ".`date`; + close(OUT); + foreach $this_id (@filename_id) { + # print OUT `cat $this_id`; + unlink("$this_id"); + } + + chomp (@filename_id = `ls *.id.pid | grep -v 'clear'`); + open (OUT, "> all.id.pid.all"); + print OUT "==Combine log at ".`date`; + print OUT `tail -n +1 *.id.pid`; + print OUT "==Log combined ".`date`; + close(OUT); + foreach $this_id (@filename_id) { + # print OUT `cat $this_id`; + unlink("$this_id"); + } + + + + chomp(@filename_id = `ls *.out | grep -v 'clear'`); + open (OUT, "> all.out.all"); + print OUT "==Combine log at ".`date`; + print OUT `tail -n +1 *.out`; + print OUT "==Log combined ".`date`; + close(OUT); + foreach $this_id (@filename_id) { + # print OUT `cat $this_id`; + unlink("$this_id"); + } + + chomp(@filename_id = `ls *.err | grep -v 'clear'`); + open (OUT, "> all.err.all"); + print OUT "==Combine log at ".`date`; + print OUT `tail -n +1 *.err`; + print OUT "==Log combined ".`date`; + close(OUT); + foreach $this_id (@filename_id) { + # print OUT `cat $this_id`; + unlink("$this_id"); + } + + # waitall.sh which cannot be deleted inside moses-parallel-sge-nosync.pl + chomp(@filename_id = `ls *waitall.sh`); + foreach $this_id (@filename_id) { + unlink("$this_id"); + } + diff --git a/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/create-config-sge-nosync.pl b/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/create-config-sge-nosync.pl new file mode 100644 index 0000000000000000000000000000000000000000..3f47cda82b9ebbe1868e09334f9fd8033a8870c9 --- /dev/null +++ b/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/create-config-sge-nosync.pl @@ -0,0 +1,426 @@ +#!/usr/bin/perl -w + +# $Id$ +# after filter-mode-given-input.pl, process the feature list + +# original code by Philipp Koehn +# changes by Ondrej Bojar +# adapted for hierarchical models by Phil Williams + +use strict; + +use FindBin qw($Bin); +use Getopt::Long; + + + +my $SCRIPTS_ROOTDIR; +if (defined($ENV{"SCRIPTS_ROOTDIR"})) { + $SCRIPTS_ROOTDIR = $ENV{"SCRIPTS_ROOTDIR"}; +} else { + $SCRIPTS_ROOTDIR = $Bin; + if ($SCRIPTS_ROOTDIR eq '') { + $SCRIPTS_ROOTDIR = dirname(__FILE__); + } + $SCRIPTS_ROOTDIR =~ s/\/training$//; + $ENV{"SCRIPTS_ROOTDIR"} = $SCRIPTS_ROOTDIR; +} + + +# moses.ini file uses FULL names for lambdas, while this training script +# internally (and on the command line) uses ABBR names. +my @ABBR_FULL_MAP = qw(d=weight-d lm=weight-l tm=weight-t w=weight-w + g=weight-generation lex=weight-lex I=weight-i); +my %ABBR2FULL = map {split/=/,$_,2} @ABBR_FULL_MAP; +my %FULL2ABBR = map {my ($a, $b) = split/=/,$_,2; ($b, $a);} @ABBR_FULL_MAP; + + + +my $verbose = 0; +my $usage = 0; # request for --help + + + + +##!# # consider phrases in input up to $MAX_LENGTH +##!# # in other words, all phrase-tables will be truncated at least to 10 words per +##!# # phrase. +##!# my $MAX_LENGTH = 10; + +# utilities +##!# my $ZCAT = "gzip -cd"; + +# get optional parameters +##!# my $opt_hierarchical = 0; +##!# my $binarizer = undef; +##!# my $opt_min_non_initial_rule_count = undef; +##!# my $opt_gzip = 1; # gzip output files (so far only phrase-based ttable until someone tests remaining models and formats) + +my $___RANGES = undef; +my $___ACTIVATE_FEATURES = undef; # comma-separated (or blank-separated) list of features to work on + # if undef work on all features + # (others are fixed to the starting values) +my $___DECODER_FLAGS = ""; # additional parametrs to pass to the decoder + +my $devbleu = undef; +my $___WORKING_DIR = undef; +my $___DEV_F = undef; +my $run = undef; # either first or final +my $runid_final = undef; +my $runid_finalplus=0; +my $sparse_weights_file = undef; + + +# set 0 if input type is text, set 1 if input type is confusion network +my $___INPUTTYPE = 0; + +my $___DECODER = undef; # required, pathname to the decoder executable +my $___CONFIG = undef; # required, pathname to startup ini file + + +GetOptions( + "activate-features=s" => \$___ACTIVATE_FEATURES, #comma-separated (or blank-separated) list of features to work on (others are fixed to the starting values) + "range=s@" => \$___RANGES, + "decoder-flags=s" => \$___DECODER_FLAGS, + "inputtype=i" => \$___INPUTTYPE, + "devbleu=s" => \$devbleu, + "sparse_weight_file=s" => \$sparse_weights_file, + "working-dir=s" => \$___WORKING_DIR, +) or exit(1); + +##!# GetOptions( +##!# "gzip!" => \$opt_gzip, +##!# "Hierarchical" => \$opt_hierarchical, +##!# "Binarizer=s" => \$binarizer, +##!# "MinNonInitialRuleCount=i" => \$opt_min_non_initial_rule_count +##!# ) or exit(1); + + +# the ?? required parameters can be supplied on the command line directly +# or using the --options +if (scalar @ARGV == 4) { + # required parameters: options + $___DEV_F = shift; + $___DECODER = shift; + $___CONFIG = shift; + $run = shift; # first or final +} + +if ($usage || !defined $___DECODER || !defined $___CONFIG) { + print STDERR "usage: $0 \$___DECODER \$___CONFIG(decoder.ini) +Options: + --activate-features=STRING ... comma-separated list of features to optimize, + others are fixed to the starting values + default: optimize all features + example: tm_0,tm_4,d_0 + --range=tm:0..1,-1..1 ... specify min and max value for some features + --range can be repeated as needed. + The order of the various --range specifications + is important only within a feature name. + E.g.: + --range=tm:0..1,-1..1 --range=tm:0..2 + is identical to: + --range=tm:0..1,-1..1,0..2 + but not to: + --range=tm:0..2 --range=tm:0..1,-1..1 + --decoder-flags=STRING ... extra parameters for the decoder + --inputtype=[0|1|2] ... Handle different input types: (0 for text, + 1 for confusion network, 2 for lattices, + default is 0) +"; + exit 1; +} + + + +##!# # get command line parameters +##!# my $dir = shift; +##!# my $config = shift; +##!# my $input = shift; + +##!# $dir = ensure_full_path($dir); + +############################################################ +############################################################ +############################################################ + +# main + +# we run moses to check validity of moses.ini and to obtain all the feature +# names + +if (($run eq "first")){ + my $featlist = get_featlist_from_moses($___CONFIG,$___CONFIG,"first"); + $featlist = insert_ranges_to_featlist($featlist, $___RANGES); + create_config($___CONFIG,"$___WORKING_DIR/run1.moses.ini",$featlist,1,(defined$devbleu?$devbleu:"--not-estimated--"),$sparse_weights_file); +} else { # $run eq "final" + chomp ($runid_final = `cat $___WORKING_DIR/finished_step.txt | tail -n 1`); + $runid_finalplus = $runid_final + 1; + `mv run${runid_finalplus}.moses.ini run_final.moses.ini`; + chomp ($devbleu = `cat $___WORKING_DIR/run_final.moses.ini | tail -n +3 | head -n 1 | gawk '{print \$3}'`); + my $featlist = get_featlist_from_moses($___CONFIG,"$___WORKING_DIR/run_final.moses.ini","final"); + $featlist = insert_ranges_to_featlist($featlist, $___RANGES); + create_config($___CONFIG,"$___WORKING_DIR/moses.ini",$featlist,$runid_finalplus,$devbleu,$sparse_weights_file); +} + +##COPIED## Mark which features are disabled: +##COPIED#if (defined $___ACTIVATE_FEATURES) { +##COPIED# my %enabled = map { ($_, 1) } split /[, ]+/, $___ACTIVATE_FEATURES; +##COPIED# my %cnt; +##COPIED# for(my $i=0; $i{"names"}}); $i++) { +##COPIED# my $name = $featlist->{"names"}->[$i]; +##COPIED# $cnt{$name} = 0 if !defined $cnt{$name}; +##COPIED# $featlist->{"enabled"}->[$i] = $enabled{$name."_".$cnt{$name}}; +##COPIED# $cnt{$name}++; +##COPIED# } +##COPIED#} else { +##COPIED# # all enabled +##COPIED# for(my $i=0; $i{"names"}}); $i++) { +##COPIED# $featlist->{"enabled"}->[$i] = 1; +##COPIED# } +##COPIED#} +##COPIED# +##COPIED#print STDERR "MERT starting values and ranges for random generation:\n"; +##COPIED#for(my $i=0; $i{"names"}}); $i++) { +##COPIED# my $name = $featlist->{"names"}->[$i]; +##COPIED# my $val = $featlist->{"values"}->[$i]; +##COPIED# my $min = $featlist->{"mins"}->[$i]; +##COPIED# my $max = $featlist->{"maxs"}->[$i]; +##COPIED# my $enabled = $featlist->{"enabled"}->[$i]; +##COPIED# printf STDERR " %5s = %7.3f", $name, $val; +##COPIED# if ($enabled) { +##COPIED# printf STDERR " (%5.2f .. %5.2f)\n", $min, $max; +##COPIED# } else { +##COPIED# print STDERR " --- inactive, not optimized ---\n"; +##COPIED# } +##COPIED#} + + + + + +sub get_featlist_from_moses { + # run moses with the given config file and return the list of features and + # their initial values + my $configfn = shift; + my $config_score = shift; + my $run = shift; + + my $featlistfn = ""; + if ($run eq 'first') { + $featlistfn = "./features.list"; # given feature list + } elsif ($run eq "final") { + $featlistfn = "./features.list.run_final"; + } + if (-e $featlistfn) { + print STDERR "Using cached features list: $featlistfn\n"; + } else { + print STDERR "Asking moses for feature names and values from $config_score\n"; + my $cmd = "$___DECODER $___DECODER_FLAGS -config $config_score -inputtype $___INPUTTYPE -show-weights > $featlistfn"; + print STDERR "$cmd\n"; #DEBUG + safesystem($cmd) or die "Failed to run moses with the config $config_score"; + } + + # read feature list + my @names = (); + my @startvalues = (); + open(INI,$featlistfn) or die "Can't read $featlistfn"; + my $nr = 0; + my @errs = (); + while () { + $nr++; + chomp; + /^(.+) (\S+) (\S+)$/ || die("invalid feature: $_"); + my ($longname, $feature, $value) = ($1,$2,$3); + next if $value eq "sparse"; + push @errs, "$featlistfn:$nr:Bad initial value of $feature: $value\n" + if $value !~ /^[+-]?[0-9.e]+$/; + push @errs, "$featlistfn:$nr:Unknown feature '$feature', please add it to \@ABBR_FULL_MAP\n" + if !defined $ABBR2FULL{$feature}; + push @names, $feature; + push @startvalues, $value; + } + close INI; + if (scalar @errs) { + print STDERR join("", @errs); + exit 1; + } + return {"names"=>\@names, "values"=>\@startvalues}; +} + + +sub insert_ranges_to_featlist { + my $featlist = shift; + my $ranges = shift; + + $ranges = [] if !defined $ranges; + + # first collect the ranges from options + my $niceranges; + foreach my $range (@$ranges) { + my $name = undef; + foreach my $namedpair (split /,/, $range) { + if ($namedpair =~ /^(.*?):/) { + $name = $1; + $namedpair =~ s/^.*?://; + die "Unrecognized name '$name' in --range=$range" + if !defined $ABBR2FULL{$name}; + } + my ($min, $max) = split /\.\./, $namedpair; + die "Bad min '$min' in --range=$range" if $min !~ /^-?[0-9.]+$/; + die "Bad max '$max' in --range=$range" if $min !~ /^-?[0-9.]+$/; + die "No name given in --range=$range" if !defined $name; + push @{$niceranges->{$name}}, [$min, $max]; + } + } + + # now populate featlist + my $seen = undef; + for(my $i=0; $i{"names"}}); $i++) { + my $name = $featlist->{"names"}->[$i]; + $seen->{$name} ++; + my $min = 0.0; + my $max = 1.0; + if (defined $niceranges->{$name}) { + my $minmax = shift @{$niceranges->{$name}}; + ($min, $max) = @$minmax if defined $minmax; + } + $featlist->{"mins"}->[$i] = $min; + $featlist->{"maxs"}->[$i] = $max; + } + return $featlist; +} + +sub safesystem { + print STDERR "Executing: @_\n"; + system(@_); + if ($? == -1) { + print STDERR "Failed to execute: @_\n $!\n"; + exit(1); + } + elsif ($? & 127) { + printf STDERR "Execution of: @_\n died with signal %d, %s coredump\n", + ($? & 127), ($? & 128) ? 'with' : 'without'; + exit(1); + } + else { + my $exitcode = $? >> 8; + print STDERR "Exit code: $exitcode\n" if $exitcode; + return ! $exitcode; + } +} + + + +sub create_config { + my $infn = shift; # source config + my $outfn = shift; # where to save the config + my $featlist = shift; # the lambdas we should write + my $iteration = shift; # just for verbosity + my $bleu_achieved = shift; # just for verbosity + my $sparse_weights_file = shift; # only defined when optimizing sparse features + + my %P; # the hash of all parameters we wish to override + + # first convert the command line parameters to the hash + { # ensure local scope of vars + my $parameter=undef; + print "Parsing --decoder-flags: |$___DECODER_FLAGS|\n"; + $___DECODER_FLAGS =~ s/^\s*|\s*$//; + $___DECODER_FLAGS =~ s/\s+/ /; + foreach (split(/ /,$___DECODER_FLAGS)) { + if (/^\-([^\d].*)$/) { + $parameter = $1; + $parameter = $ABBR2FULL{$parameter} if defined($ABBR2FULL{$parameter}); + } + else { + die "Found value with no -paramname before it: $_" + if !defined $parameter; + push @{$P{$parameter}},$_; + } + } + } + + # First delete all weights params from the input, we're overwriting them. + # Delete both short and long-named version. + for(my $i=0; $i{"names"}}); $i++) { + my $name = $featlist->{"names"}->[$i]; + delete($P{$name}); + delete($P{$ABBR2FULL{$name}}); + } + + # Convert weights to elements in P + for(my $i=0; $i{"names"}}); $i++) { + my $name = $featlist->{"names"}->[$i]; + my $val = $featlist->{"values"}->[$i]; + $name = defined $ABBR2FULL{$name} ? $ABBR2FULL{$name} : $name; + # ensure long name + push @{$P{$name}}, $val; + } + + if (defined($sparse_weights_file)) { + push @{$P{"weights-file"}}, $___WORKING_DIR."/".$sparse_weights_file; + } + + # create new moses.ini decoder config file by cloning and overriding the original one + open(INI,$infn) or die "Can't read $infn"; + delete($P{"config"}); # never output + print "Saving new config to: $outfn\n"; + open(OUT,"> $outfn") or die "Can't write $outfn"; + print OUT "# MERT optimized configuration\n"; + print OUT "# decoder $___DECODER\n"; + print OUT "# BLEU $bleu_achieved on dev $___DEV_F\n"; + print OUT "# We were before running iteration $iteration\n"; + print OUT "# finished ".`date`; + my $line = ; + while(1) { + last unless $line; + + # skip until hit [parameter] + if ($line !~ /^\[(.+)\]\s*$/) { + $line = ; + print OUT $line if $line =~ /^\#/ || $line =~ /^\s+$/; + next; + } + + # parameter name + my $parameter = $1; + $parameter = $ABBR2FULL{$parameter} if defined($ABBR2FULL{$parameter}); + print OUT "[$parameter]\n"; + + # change parameter, if new values + if (defined($P{$parameter})) { + # write new values + foreach (@{$P{$parameter}}) { + print OUT $_."\n"; + } + delete($P{$parameter}); + # skip until new parameter, only write comments + while($line = ) { + print OUT $line if $line =~ /^\#/ || $line =~ /^\s+$/; + last if $line =~ /^\[/; + last unless $line; + } + next; + } + # unchanged parameter, write old + while($line = ) { + last if $line =~ /^\[/; + print OUT $line; + } + } + + # write all additional parameters + foreach my $parameter (keys %P) { + print OUT "\n[$parameter]\n"; + foreach (@{$P{$parameter}}) { + print OUT $_."\n"; + } + } + + close(INI); + close(OUT); + print STDERR "Saved: $outfn\n"; +} + + diff --git a/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/moses-parallel-postdecode-sge-nosync.pl b/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/moses-parallel-postdecode-sge-nosync.pl new file mode 100644 index 0000000000000000000000000000000000000000..0edbe37dbfd752035694b693c1e25ad740733432 --- /dev/null +++ b/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/moses-parallel-postdecode-sge-nosync.pl @@ -0,0 +1,235 @@ +#!/usr/bin/perl + + +my $logflag=""; +my $logfile=""; +my $alifile=undef; +my $nbestflag=0; +my $processid=0; +my $idxliststr=""; +my $workingdir=""; +my $inputfile=""; +my $tmpdir=""; +my $splitpfx=""; +my $jobscript=""; +my $qsubout=""; +my $qsuberr=""; +my $nbestfile=undef; +my $nbestlist=undef; +my $outnbest=""; +my $lsamp_filename=""; +my @idxlist=(); + + +############################### +# Script starts here + +init(); + + + +#concatenating translations and removing temporary files +concatenate_1best(); +concatenate_logs() if $logflag; +concatenate_ali() if defined $alifile; +concatenate_nbest() if $nbestflag; +safesystem("cat nbest$$ >> /dev/stdout") if $nbestlist[0] eq '-'; + + +print STDERR "Not support searchgraphflag for sync mert\n" if $searchgraphflag; +# concatenate_searchgraph() if $searchgraphflag; +# safesystem("cat searchgraph$$ >> /dev/stdout") if $searchgraphlist eq '-'; + +print STDERR "Not support wordgraphflag for sync mert\n" if $searchgraphflag; +# concatenate_wordgraph() if $wordgraphflag; +# safesystem("cat wordgraph$$ >> /dev/stdout") if $wordgraphlist[0] eq '-'; + +remove_temporary_files(); +#### +#### ### ending scripts in run_decoder() ############## +#### sanity_check_order_of_lambdas($featlist, $filename); +#### ## how to do return??? +#### return ($filename, $lsamp_filename); +###################################################### + + + + +sub init(){ + use strict; + use Getopt::Long qw(:config pass_through no_ignore_case permute); + + GetOptions('alignment-output-file=s'=>\$alifile, + 'process-id=s'=>\$processid, + 'idxliststr=s'=>\$idxliststr, + 'logfile=s'=>\$logfile, + 'nbestfile=s'=>\$nbestfile, + 'outnbest=s'=>\$outnbest, + 'lsamp-filename=s'=>\$lsamp_filename, + 'input-file=s'=>\$inputfile + ) or exit(1); + + if ($logfile){ $logflag=1; } + + if (defined $nbestfile) { $nbestflag=1; } + + $idxliststr =~ s/^\s+|\s+$//g; + @idxlist = split(/\s+/,$idxliststr); + + my $pwdcmd = getPwdCmd(); + + $workingdir = `$pwdcmd`; chomp $workingdir; + $tmpdir="$workingdir/tmp$processid"; + $splitpfx="split$processid"; + + $jobscript="$workingdir/job$processid"; + $qsubout="$workingdir/out.job$processid"; + $qsuberr="$workingdir/err.job$processid"; + + # print STDERR "$idxliststr\n"; + +} + + +sub concatenate_nbest(){ + my $oldcode=""; + my $newcode=-1; + my %inplength = (); + my $offset = 0; + +# get the list of feature and set a fictitious string with zero scores + open (IN, "${nbestfile}.${splitpfx}$idxlist[0]"); + my $str = ; + chomp($str); + close(IN); + my ($code,$trans,$featurescores,$globalscore)=split(/\|\|\|/,$str); + + my $emptytrans = " "; + my $emptyglobalscore = " 0.0"; + my $emptyfeaturescores = $featurescores; + $emptyfeaturescores =~ s/[-0-9\.]+/0/g; + + if ($outnbest eq '-'){ $outnbest="nbest$processid"; } + + # my $outnbest=$nbestlist[0]; + # if ($nbestlist[0] eq '-'){ $outnbest="nbest$$"; } + + open (OUT, "> $outnbest"); + foreach my $idx (@idxlist){ + +#computing the length of each input file + # print STDERR "this idx: $idx\n"; + + my @in=(); + open (IN, "${inputfile}.${splitpfx}${idx}.trans"); + @in=; + close(IN); + $inplength{$idx} = scalar(@in); + + open (IN, "${nbestfile}.${splitpfx}${idx}"); + while (){ + my ($code,@extra)=split(/\|\|\|/,$_); + $code += $offset; + if ($code ne $oldcode){ +# if there is a jump between two consecutive codes +# it means that an input sentence is not translated +# fill this hole with a "fictitious" list of translation +# comprising just one "emtpy translation" with zero scores + while ($code - $oldcode > 1){ + $oldcode++; + print OUT join("\|\|\|",($oldcode,$emptytrans,$emptyfeaturescores,$emptyglobalscore)),"\n"; + } + } + $oldcode=$code; + print OUT join("\|\|\|",($oldcode,@extra)); + } + close(IN); + $offset += $inplength{$idx}; + + while ($offset - $oldcode > 1){ + $oldcode++; + print OUT join("\|\|\|",($oldcode,$emptytrans,$emptyfeaturescores,$emptyglobalscore)),"\n"; + } + } + close(OUT); +} + + +sub concatenate_1best(){ + foreach my $idx (@idxlist){ + # print STDERR "reading 1best file ${inputfile}.${splitpfx}$idx.trans\n"; + my @in=(); + open (IN, "${inputfile}.${splitpfx}${idx}.trans"); + @in=; + # print STDERR "in array is : @in"; + print STDOUT "@in"; + close(IN); + } +} + +sub concatenate_logs(){ + open (OUT, "> ${logfile}"); + foreach my $idx (@idxlist){ + my @in=(); + open (IN, "$qsubout$idx"); + @in=; + print OUT "@in"; + close(IN); + } + close(OUT); +} + +sub concatenate_ali(){ + open (OUT, "> ${alifile}"); + foreach my $idx (@idxlist){ + my @in=(); + open (IN, "$alifile.$splitpfx$idx"); + @in=; + print OUT "@in"; + close(IN); + } + close(OUT); +} + + + + +# look for the correct pwdcmd (pwd by default, pawd if it exists) +# I assume that pwd always exists +sub getPwdCmd(){ + my $pwdcmd="pwd"; + my $a; + chomp($a=`which pawd | head -1 | awk '{print $1}'`); + if ($a && -e $a){ $pwdcmd=$a; } + return $pwdcmd; +} + + +sub remove_temporary_files(){ + # removing temporary files + foreach my $idx (@idxlist){ + unlink("${inputfile}.${splitpfx}${idx}.trans"); + unlink("${inputfile}.${splitpfx}${idx}"); + if (defined $alifile){ unlink("${alifile}.${splitpfx}${idx}"); } + if ($nbestflag){ unlink("${nbestfile}.${splitpfx}${idx}"); } + if ($searchgraphflag){ unlink("${searchgraphfile}.${splitpfx}${idx}"); } + if ($wordgraphflag){ unlink("${wordgraphfile}.${splitpfx}${idx}"); } + + # print STDERR "Deleting ${jobscript}${idx}.bash\n"; + unlink("${jobscript}${idx}.bash"); + unlink("${jobscript}${idx}.log"); + unlink("$qsubname.W.log"); + unlink("$qsubout$idx"); + unlink("$qsuberr$idx"); + rmdir("$tmpdir"); + } + # unlink("${jobscript}.sync_workaround_script.sh"); + if ($nbestflag && $nbestlist[0] eq '-'){ unlink("${nbestfile}$$"); }; + if ($searchgraphflag && $searchgraphlist eq '-'){ unlink("${searchgraphfile}$$"); }; + if ($wordgraphflag && $wordgraphlist eq '-'){ unlink("${wordgraphfile}$$"); }; +} + + + + + diff --git a/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/poll-decoder.pl b/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/poll-decoder.pl new file mode 100644 index 0000000000000000000000000000000000000000..54e9ecedac60d521bdc81e6df4d4aba8fddfbdaf --- /dev/null +++ b/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/poll-decoder.pl @@ -0,0 +1,30 @@ +#!/usr/bin/perl + +use Getopt::Long qw(:config pass_through no_ignore_case permute); + +my $poll_target = undef; +my $working_dir = undef; + +GetOptions('poll-target=s'=> \$poll_target, + 'working-dir'=> \$working_dir + ) or exit(1); + + +if (defined $working_dir) { + chdir($working_dir); +} + +my $cnt = 1; + +print STDERR "Wait for file: $poll_target\n"; + +while (1) { + if (-e $poll_target){ + print STDERR "\n File found!!\n"; + last; + } else { + sleep(10); + print STDERR "."; + } +} + diff --git a/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/process-featlist-sge-nosync.pl b/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/process-featlist-sge-nosync.pl new file mode 100644 index 0000000000000000000000000000000000000000..02b18526a3660bdb1f038693bf1fd15db957f58e --- /dev/null +++ b/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/process-featlist-sge-nosync.pl @@ -0,0 +1,283 @@ +#!/usr/bin/perl -w + +# $Id$ +# after filter-mode-given-input.pl, process the feature list + +# original code by Philipp Koehn +# changes by Ondrej Bojar +# adapted for hierarchical models by Phil Williams + +use strict; + +use FindBin qw($Bin); +use Getopt::Long; + + + +my $SCRIPTS_ROOTDIR; +if (defined($ENV{"SCRIPTS_ROOTDIR"})) { + $SCRIPTS_ROOTDIR = $ENV{"SCRIPTS_ROOTDIR"}; +} else { + $SCRIPTS_ROOTDIR = $Bin; + if ($SCRIPTS_ROOTDIR eq '') { + $SCRIPTS_ROOTDIR = dirname(__FILE__); + } + $SCRIPTS_ROOTDIR =~ s/\/training$//; + $ENV{"SCRIPTS_ROOTDIR"} = $SCRIPTS_ROOTDIR; +} + + +# moses.ini file uses FULL names for lambdas, while this training script +# internally (and on the command line) uses ABBR names. +my @ABBR_FULL_MAP = qw(d=weight-d lm=weight-l tm=weight-t w=weight-w + g=weight-generation lex=weight-lex I=weight-i); +my %ABBR2FULL = map {split/=/,$_,2} @ABBR_FULL_MAP; +my %FULL2ABBR = map {my ($a, $b) = split/=/,$_,2; ($b, $a);} @ABBR_FULL_MAP; + + + +my $verbose = 0; +my $usage = 0; # request for --help + + + + +##!# # consider phrases in input up to $MAX_LENGTH +##!# # in other words, all phrase-tables will be truncated at least to 10 words per +##!# # phrase. +##!# my $MAX_LENGTH = 10; + +# utilities +##!# my $ZCAT = "gzip -cd"; + +# get optional parameters +##!# my $opt_hierarchical = 0; +##!# my $binarizer = undef; +##!# my $opt_min_non_initial_rule_count = undef; +##!# my $opt_gzip = 1; # gzip output files (so far only phrase-based ttable until someone tests remaining models and formats) + +my $___RANGES = undef; +my $___ACTIVATE_FEATURES = undef; # comma-separated (or blank-separated) list of features to work on + # if undef work on all features + # (others are fixed to the starting values) +my $___DECODER_FLAGS = ""; # additional parametrs to pass to the decoder + +# set 0 if input type is text, set 1 if input type is confusion network +my $___INPUTTYPE = 0; + +my $___DECODER = undef; # required, pathname to the decoder executable +my $___CONFIG = undef; # required, pathname to startup ini file + + +GetOptions( + "activate-features=s" => \$___ACTIVATE_FEATURES, #comma-separated (or blank-separated) list of features to work on (others are fixed to the starting values) + "range=s@" => \$___RANGES, + "decoder-flags=s" => \$___DECODER_FLAGS, + "inputtype=i" => \$___INPUTTYPE +) or exit(1); + +##!# GetOptions( +##!# "gzip!" => \$opt_gzip, +##!# "Hierarchical" => \$opt_hierarchical, +##!# "Binarizer=s" => \$binarizer, +##!# "MinNonInitialRuleCount=i" => \$opt_min_non_initial_rule_count +##!# ) or exit(1); + + +# the ?? required parameters can be supplied on the command line directly +# or using the --options +if (scalar @ARGV == 2) { + # required parameters: options + $___DECODER = shift; + $___CONFIG = shift; +} + +if ($usage || !defined $___DECODER || !defined $___CONFIG) { + print STDERR "usage: $0 \$___DECODER \$___CONFIG(decoder.ini) +Options: + --activate-features=STRING ... comma-separated list of features to optimize, + others are fixed to the starting values + default: optimize all features + example: tm_0,tm_4,d_0 + --range=tm:0..1,-1..1 ... specify min and max value for some features + --range can be repeated as needed. + The order of the various --range specifications + is important only within a feature name. + E.g.: + --range=tm:0..1,-1..1 --range=tm:0..2 + is identical to: + --range=tm:0..1,-1..1,0..2 + but not to: + --range=tm:0..2 --range=tm:0..1,-1..1 + --decoder-flags=STRING ... extra parameters for the decoder + --inputtype=[0|1|2] ... Handle different input types: (0 for text, + 1 for confusion network, 2 for lattices, + default is 0) +"; + exit 1; +} + + + +##!# # get command line parameters +##!# my $dir = shift; +##!# my $config = shift; +##!# my $input = shift; + +##!# $dir = ensure_full_path($dir); + +############################################################ +############################################################ +############################################################ + +# main + +# we run moses to check validity of moses.ini and to obtain all the feature +# names +my $featlist = get_featlist_from_moses($___CONFIG); +$featlist = insert_ranges_to_featlist($featlist, $___RANGES); + + +# Mark which features are disabled: +if (defined $___ACTIVATE_FEATURES) { + my %enabled = map { ($_, 1) } split /[, ]+/, $___ACTIVATE_FEATURES; + my %cnt; + for(my $i=0; $i{"names"}}); $i++) { + my $name = $featlist->{"names"}->[$i]; + $cnt{$name} = 0 if !defined $cnt{$name}; + $featlist->{"enabled"}->[$i] = $enabled{$name."_".$cnt{$name}}; + $cnt{$name}++; + } +} else { + # all enabled + for(my $i=0; $i{"names"}}); $i++) { + $featlist->{"enabled"}->[$i] = 1; + } +} + +print STDERR "MERT starting values and ranges for random generation:\n"; +for(my $i=0; $i{"names"}}); $i++) { + my $name = $featlist->{"names"}->[$i]; + my $val = $featlist->{"values"}->[$i]; + my $min = $featlist->{"mins"}->[$i]; + my $max = $featlist->{"maxs"}->[$i]; + my $enabled = $featlist->{"enabled"}->[$i]; + printf STDERR " %5s = %7.3f", $name, $val; + if ($enabled) { + printf STDERR " (%5.2f .. %5.2f)\n", $min, $max; + } else { + print STDERR " --- inactive, not optimized ---\n"; + } +} + + + + + +sub get_featlist_from_moses { + # run moses with the given config file and return the list of features and + # their initial values + my $configfn = shift; + my $featlistfn = "./features.list"; + if (-e $featlistfn) { + print STDERR "Using cached features list: $featlistfn\n"; + } else { + print STDERR "Asking moses for feature names and values from $___CONFIG\n"; + my $cmd = "$___DECODER $___DECODER_FLAGS -config $configfn -inputtype $___INPUTTYPE -show-weights > $featlistfn"; + print STDERR "$cmd\n"; #DEBUG + safesystem($cmd) or die "Failed to run moses with the config $configfn"; + } + + # read feature list + my @names = (); + my @startvalues = (); + open(INI,$featlistfn) or die "Can't read $featlistfn"; + my $nr = 0; + my @errs = (); + while () { + $nr++; + chomp; + /^(.+) (\S+) (\S+)$/ || die("invalid feature: $_"); + my ($longname, $feature, $value) = ($1,$2,$3); + next if $value eq "sparse"; + push @errs, "$featlistfn:$nr:Bad initial value of $feature: $value\n" + if $value !~ /^[+-]?[0-9.e]+$/; + push @errs, "$featlistfn:$nr:Unknown feature '$feature', please add it to \@ABBR_FULL_MAP\n" + if !defined $ABBR2FULL{$feature}; + push @names, $feature; + push @startvalues, $value; + } + close INI; + if (scalar @errs) { + print STDERR join("", @errs); + exit 1; + } + return {"names"=>\@names, "values"=>\@startvalues}; +} + + +sub insert_ranges_to_featlist { + my $featlist = shift; + my $ranges = shift; + + $ranges = [] if !defined $ranges; + + # first collect the ranges from options + my $niceranges; + foreach my $range (@$ranges) { + my $name = undef; + foreach my $namedpair (split /,/, $range) { + if ($namedpair =~ /^(.*?):/) { + $name = $1; + $namedpair =~ s/^.*?://; + die "Unrecognized name '$name' in --range=$range" + if !defined $ABBR2FULL{$name}; + } + my ($min, $max) = split /\.\./, $namedpair; + die "Bad min '$min' in --range=$range" if $min !~ /^-?[0-9.]+$/; + die "Bad max '$max' in --range=$range" if $min !~ /^-?[0-9.]+$/; + die "No name given in --range=$range" if !defined $name; + push @{$niceranges->{$name}}, [$min, $max]; + } + } + + # now populate featlist + my $seen = undef; + for(my $i=0; $i{"names"}}); $i++) { + my $name = $featlist->{"names"}->[$i]; + $seen->{$name} ++; + my $min = 0.0; + my $max = 1.0; + if (defined $niceranges->{$name}) { + my $minmax = shift @{$niceranges->{$name}}; + ($min, $max) = @$minmax if defined $minmax; + } + $featlist->{"mins"}->[$i] = $min; + $featlist->{"maxs"}->[$i] = $max; + } + return $featlist; +} + +sub safesystem { + print STDERR "Executing: @_\n"; + system(@_); + if ($? == -1) { + print STDERR "Failed to execute: @_\n $!\n"; + exit(1); + } + elsif ($? & 127) { + printf STDERR "Execution of: @_\n died with signal %d, %s coredump\n", + ($? & 127), ($? & 128) ? 'with' : 'without'; + exit(1); + } + else { + my $exitcode = $? >> 8; + print STDERR "Exit code: $exitcode\n" if $exitcode; + return ! $exitcode; + } +} + + + + + diff --git a/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/process-moses-result-sge-nosync.pl b/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/process-moses-result-sge-nosync.pl new file mode 100644 index 0000000000000000000000000000000000000000..ee37b0f8f608423556d4f774d94ef8d3d08c41ab --- /dev/null +++ b/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/process-moses-result-sge-nosync.pl @@ -0,0 +1,1377 @@ +#! /usr/bin/perl + +# $Id$ +# Usage: +# mert-moses.pl +# For other options see below or run 'mert-moses.pl --help' + +# Notes: +# and should be raw text files, one sentence per line +# can be a prefix, in which case the files are 0, 1, etc. are used + +# Excerpts from revision history + +# Sept 2011 multi-threaded mert (Barry Haddow) +# 3 Aug 2011 Added random directions, historic best, pairwise ranked (PK) +# Jul 2011 simplifications (Ondrej Bojar) +# -- rely on moses' -show-weights instead of parsing moses.ini +# ... so moses is also run once *before* mert starts, checking +# the model to some extent +# -- got rid of the 'triples' mess; +# use --range to supply bounds for random starting values: +# --range tm:-3..3 --range lm:-3..3 +# 5 Aug 2009 Handling with different reference length policies (shortest, average, closest) for BLEU +# and case-sensistive/insensitive evaluation (Nicola Bertoldi) +# 5 Jun 2008 Forked previous version to support new mert implementation. +# 13 Feb 2007 Better handling of default values for lambda, now works with multiple +# models and lexicalized reordering +# 11 Oct 2006 Handle different input types through parameter --inputype=[0|1] +# (0 for text, 1 for confusion network, default is 0) (Nicola Bertoldi) +# 10 Oct 2006 Allow skip of filtering of phrase tables (--no-filter-phrase-table) +# useful if binary phrase tables are used (Nicola Bertoldi) +# 28 Aug 2006 Use either closest or average or shortest (default) reference +# length as effective reference length +# Use either normalization or not (default) of texts (Nicola Bertoldi) +# 31 Jul 2006 move gzip run*.out to avoid failure wit restartings +# adding default paths +# 29 Jul 2006 run-filter, score-nbest and mert run on the queue (Nicola; Ondrej had to type it in again) +# 28 Jul 2006 attempt at foolproof usage, strong checking of input validity, merged the parallel and nonparallel version (Ondrej Bojar) +# 27 Jul 2006 adding the safesystem() function to handle with process failure +# 22 Jul 2006 fixed a bug about handling relative path of configuration file (Nicola Bertoldi) +# 21 Jul 2006 adapted for Moses-in-parallel (Nicola Bertoldi) +# 18 Jul 2006 adapted for Moses and cleaned up (PK) +# 21 Jan 2005 unified various versions, thorough cleanup (DWC) +# now indexing accumulated n-best list solely by feature vectors +# 14 Dec 2004 reimplemented find_threshold_points in C (NMD) +# 25 Oct 2004 Use either average or shortest (default) reference +# length as effective reference length (DWC) +# 13 Oct 2004 Use alternative decoders (DWC) +# Original version by Philipp Koehn + +use strict; +use Net::OpenSSH::Compat::Perl; + +use FindBin qw($Bin); +use File::Basename; +use File::Path; +use File::Spec; +use Cwd; + +my $SCRIPTS_ROOTDIR = $Bin; +$SCRIPTS_ROOTDIR =~ s/\/training$//; +$SCRIPTS_ROOTDIR = $ENV{"SCRIPTS_ROOTDIR"} if defined($ENV{"SCRIPTS_ROOTDIR"}); + +## We preserve this bit of comments to keep the traditional weight ranges. +# "w" => [ [ 0.0, -1.0, 1.0 ] ], # word penalty +# "d" => [ [ 1.0, 0.0, 2.0 ] ], # lexicalized reordering model +# "lm" => [ [ 1.0, 0.0, 2.0 ] ], # language model +# "g" => [ [ 1.0, 0.0, 2.0 ], # generation model +# [ 1.0, 0.0, 2.0 ] ], +# "tm" => [ [ 0.3, 0.0, 0.5 ], # translation model +# [ 0.2, 0.0, 0.5 ], +# [ 0.3, 0.0, 0.5 ], +# [ 0.2, 0.0, 0.5 ], +# [ 0.0,-1.0, 1.0 ] ], # ... last weight is phrase penalty +# "lex"=> [ [ 0.1, 0.0, 0.2 ] ], # global lexical model +# "I" => [ [ 0.0,-1.0, 1.0 ] ], # input lattice scores + + + +# moses.ini file uses FULL names for lambdas, while this training script +# internally (and on the command line) uses ABBR names. +my @ABBR_FULL_MAP = qw(d=weight-d lm=weight-l tm=weight-t w=weight-w + g=weight-generation lex=weight-lex I=weight-i); +my %ABBR2FULL = map {split/=/,$_,2} @ABBR_FULL_MAP; +my %FULL2ABBR = map {my ($a, $b) = split/=/,$_,2; ($b, $a);} @ABBR_FULL_MAP; + +my $minimum_required_change_in_weights = 0.00001; + # stop if no lambda changes more than this + +my $verbose = 0; +my $usage = 0; # request for --help + +# We assume that if you don't specify working directory, +# we set the default is set to `pwd`/mert-work +# my $___WORKING_DIR = File::Spec->catfile(Cwd::getcwd(), "mert-work"); +my $___WORKING_DIR = undef; +my $___DEV_E = undef; +my $___DEV_F = undef; # required, input text to decode +my $___DECODER = undef; # required, pathname to the decoder executable +my $___CONFIG = undef; # required, pathname to startup ini file +my $___N_BEST_LIST_SIZE = 100; +my $___LATTICE_SAMPLES = 0; +my $queue_flags = "-hard"; # extra parameters for parallelizer + # the -l ws0ssmt was relevant only to JHU 2006 workshop +my $___JOBS = undef; # if parallel, number of jobs to use (undef or 0 -> serial) +my $___DECODER_FLAGS = ""; # additional parametrs to pass to the decoder +my $continue = 0; # should we try to continue from the last saved step? +my $skip_decoder = 0; # and should we skip the first decoder run (assuming we got interrupted during mert) +my $___FILTER_PHRASE_TABLE = 1; # filter phrase table +my $___PREDICTABLE_SEEDS = 0; +my $___START_WITH_HISTORIC_BESTS = 0; # use best settings from all previous iterations as starting points [Foster&Kuhn,2009] +my $___RANDOM_DIRECTIONS = 0; # search in random directions only +my $___NUM_RANDOM_DIRECTIONS = 0; # number of random directions, also works with default optimizer [Cer&al.,2008] +my $___PAIRWISE_RANKED_OPTIMIZER = 0; # use Hopkins&May[2011] +my $___PRO_STARTING_POINT = 0; # get a starting point from pairwise ranked optimizer +my $___RANDOM_RESTARTS = 20; +my $___HISTORIC_INTERPOLATION = 0; # interpolate optimize weights with previous iteration's weights [Hopkins&May,2011,5.4.3] +my $__THREADS = 0; +my $run = 0; +my $nextrun = 0; +my $submithost = ""; + + +# Parameter for effective reference length when computing BLEU score +# Default is to use shortest reference +# Use "--shortest" to use shortest reference length +# Use "--average" to use average reference length +# Use "--closest" to use closest reference length +# Only one between --shortest, --average and --closest can be set +# If more than one choice the defualt (--shortest) is used +my $___SHORTEST = 0; +my $___AVERAGE = 0; +my $___CLOSEST = 0; + +# Use "--nocase" to compute case-insensitive scores +my $___NOCASE = 0; + +# Use "--nonorm" to non normalize translation before computing scores +my $___NONORM = 0; + +# set 0 if input type is text, set 1 if input type is confusion network +my $___INPUTTYPE = 0; + + +my $mertdir = undef; # path to new mert directory +my $mertargs = undef; # args to pass through to mert & extractor +my $mertmertargs = undef; # args to pass through to mert only +my $extractorargs = undef; # args to pass through to extractor only +my $filtercmd = undef; # path to filter-model-given-input.pl +my $filterfile = undef; +my $qsubwrapper = undef; +my $qsubwrapper_exit = undef; +my $moses_parallel_cmd = undef; +my $old_sge = 0; # assume sge<6.0 +my $___CONFIG_ORIG = undef; # pathname to startup ini file before filtering +my $___ACTIVATE_FEATURES = undef; # comma-separated (or blank-separated) list of features to work on + # if undef work on all features + # (others are fixed to the starting values) +my $___RANGES = undef; +my $prev_aggregate_nbl_size = -1; # number of previous step to consider when loading data (default =-1) + # -1 means all previous, i.e. from iteration 1 + # 0 means no previous data, i.e. from actual iteration + # 1 means 1 previous data , i.e. from the actual iteration and from the previous one + # and so on +my $maximum_iterations = 25; + +##################### +my $processfeatlistcmd = undef; +my $processfeatlistargs = undef; +my $createconfigcmd = undef; +my $createconfigargs = undef; +my $decoderargs = undef; +##################### + +use Getopt::Long; +GetOptions( + "working-dir=s" => \$___WORKING_DIR, + "input=s" => \$___DEV_F, + "inputtype=i" => \$___INPUTTYPE, + "refs=s" => \$___DEV_E, + "decoder=s" => \$___DECODER, + "config=s" => \$___CONFIG, + "nbest=i" => \$___N_BEST_LIST_SIZE, + "lattice-samples=i" => \$___LATTICE_SAMPLES, + "queue-flags=s" => \$queue_flags, + "jobs=i" => \$___JOBS, + "decoder-flags=s" => \$___DECODER_FLAGS, + "continue" => \$continue, + "skip-decoder" => \$skip_decoder, + "shortest" => \$___SHORTEST, + "average" => \$___AVERAGE, + "closest" => \$___CLOSEST, + "nocase" => \$___NOCASE, + "nonorm" => \$___NONORM, + "help" => \$usage, + "verbose" => \$verbose, + "mertdir=s" => \$mertdir, + "mertargs=s" => \$mertargs, + "extractorargs=s" => \$extractorargs, + "mertmertargs=s" => \$mertmertargs, + "rootdir=s" => \$SCRIPTS_ROOTDIR, + "filtercmd=s" => \$filtercmd, # allow to override the default location + "filterfile=s" => \$filterfile, # input to filtering script (useful for lattices/confnets) + "qsubwrapper=s" => \$qsubwrapper, # allow to override the default location + "mosesparallelcmd=s" => \$moses_parallel_cmd, # allow to override the default location + "old-sge" => \$old_sge, #passed to moses-parallel + "filter-phrase-table!" => \$___FILTER_PHRASE_TABLE, # (dis)allow of phrase tables + "predictable-seeds" => \$___PREDICTABLE_SEEDS, # make random restarts deterministic + "historic-bests" => \$___START_WITH_HISTORIC_BESTS, # use best settings from all previous iterations as starting points + "random-directions" => \$___RANDOM_DIRECTIONS, # search only in random directions + "run=i" => \$run, + "number-of-random-directions=i" => \$___NUM_RANDOM_DIRECTIONS, # number of random directions + "random-restarts=i" => \$___RANDOM_RESTARTS, # number of random restarts + "activate-features=s" => \$___ACTIVATE_FEATURES, #comma-separated (or blank-separated) list of features to work on (others are fixed to the starting values) + "range=s@" => \$___RANGES, + "submithost=s" => \$submithost, + "prev-aggregate-nbestlist=i" => \$prev_aggregate_nbl_size, #number of previous step to consider when loading data (default =-1, i.e. all previous) + "maximum-iterations=i" => \$maximum_iterations, + "pairwise-ranked" => \$___PAIRWISE_RANKED_OPTIMIZER, + "pro-starting-point" => \$___PRO_STARTING_POINT, + "historic-interpolation=f" => \$___HISTORIC_INTERPOLATION, + "threads=i" => \$__THREADS +) or exit(1); + +# the 4 required parameters can be supplied on the command line directly +# or using the --options +if (scalar @ARGV == 3) { + # required parameters: input_file references_basename decoder_executable + $___DEV_F = shift; + $___DECODER = shift; + $___CONFIG = shift; +} + + + +if ($usage || !defined $___DECODER || !defined $___CONFIG || !defined $___DEV_F) { + print STDERR "usage: $0 input-text decoder-executable decoder.ini +Options: + --working-dir=mert-dir ... where all the files are created + --nbest=100 ... how big nbestlist to generate + --lattice-samples ... how many lattice samples (Chatterjee & Cancedda, emnlp 2010) + --jobs=N ... set this to anything to run moses in parallel + --mosesparallelcmd=STR ... use a different script instead of moses-parallel + --queue-flags=STRING ... anything you with to pass to qsub, eg. + '-l ws06osssmt=true'. The default is: '-hard' + To reset the parameters, please use + --queue-flags=' ' + (i.e. a space between the quotes). + --decoder-flags=STRING ... extra parameters for the decoder + --continue ... continue from the last successful iteration + --skip-decoder ... skip the decoder run for the first time, + assuming that we got interrupted during + optimization + --shortest --average --closest + ... Use shortest/average/closest reference length + as effective reference length (mutually exclusive) + --nocase ... Do not preserve case information; i.e. + case-insensitive evaluation (default is false). + --nonorm ... Do not use text normalization (flag is not active, + i.e. text is NOT normalized) + --filtercmd=STRING ... path to filter-model-given-input.pl + --filterfile=STRING ... path to alternative to input-text for filtering + model. useful for lattice decoding + --rootdir=STRING ... where do helpers reside (if not given explicitly) + --mertdir=STRING ... path to new mert implementation + --mertargs=STRING ... extra args for both extractor and mert + --extractorargs=STRING ... extra args for extractor only + --mertmertargs=STRING ... extra args for mert only + --scorenbestcmd=STRING ... path to score-nbest.py + --old-sge ... passed to parallelizers, assume Grid Engine < 6.0 + --inputtype=[0|1|2] ... Handle different input types: (0 for text, + 1 for confusion network, 2 for lattices, + default is 0) + --no-filter-phrase-table ... disallow filtering of phrase tables + (useful if binary phrase tables are available) + --random-restarts=INT ... number of random restarts (default: 20) + --predictable-seeds ... provide predictable seeds to mert so that random + restarts are the same on every run + --range=tm:0..1,-1..1 ... specify min and max value for some features + --range can be repeated as needed. + The order of the various --range specifications + is important only within a feature name. + E.g.: + --range=tm:0..1,-1..1 --range=tm:0..2 + is identical to: + --range=tm:0..1,-1..1,0..2 + but not to: + --range=tm:0..2 --range=tm:0..1,-1..1 + --activate-features=STRING ... comma-separated list of features to optimize, + others are fixed to the starting values + default: optimize all features + example: tm_0,tm_4,d_0 + --prev-aggregate-nbestlist=INT ... number of previous step to consider when + loading data (default = $prev_aggregate_nbl_size) + -1 means all previous, i.e. from iteration 1 + 0 means no previous data, i.e. only the + current iteration + N means this and N previous iterations + + --maximum-iterations=ITERS ... Maximum number of iterations. Default: $maximum_iterations + --random-directions ... search only in random directions + --number-of-random-directions=int ... number of random directions + (also works with regular optimizer, default: 0) + --pairwise-ranked ... Use PRO for optimisation (Hopkins and May, emnlp 2011) + --pro-starting-point ... Use PRO to get a starting point for MERT + --threads=NUMBER ... Use multi-threaded mert (must be compiled in). + --historic-interpolation ... Interpolate optimized weights with prior iterations' weight + (parameter sets factor [0;1] given to current weights) +"; + exit 1; +} + + +# Check validity of input parameters and set defaults if needed + +print STDERR "Using SCRIPTS_ROOTDIR: $SCRIPTS_ROOTDIR\n"; + +# path of script for filtering phrase tables and running the decoder +$filtercmd="$SCRIPTS_ROOTDIR/training/filter-model-given-input.pl" if !defined $filtercmd; + +if ( ! -x $filtercmd && ! $___FILTER_PHRASE_TABLE) { + print STDERR "Filtering command not found: $filtercmd.\n"; + print STDERR "Use --filtercmd=PATH to specify a valid one or --no-filter-phrase-table\n"; + exit 1; +} + +# $qsubwrapper="$SCRIPTS_ROOTDIR/generic/qsub-wrapper.pl" if !defined $qsubwrapper; +$qsubwrapper = "$SCRIPTS_ROOTDIR/generic/qsub-wrapper-sge-nosync.pl" if !defined $qsubwrapper; + +$qsubwrapper_exit = "$SCRIPTS_ROOTDIR/generic/qsub-wrapper-exit-sge-nosync.pl" if !defined $qsubwrapper_exit; + +# $moses_parallel_cmd = "$SCRIPTS_ROOTDIR/generic/moses-parallel.pl" +# if !defined $moses_parallel_cmd; +$moses_parallel_cmd = "$SCRIPTS_ROOTDIR/generic/moses-parallel-sge-nosync.pl" + if !defined $moses_parallel_cmd; + +if (!defined $mertdir) { + $mertdir = "$SCRIPTS_ROOTDIR/../mert"; + print STDERR "Assuming --mertdir=$mertdir\n"; +} + +my $mert_extract_cmd = "$mertdir/extractor"; +my $mert_mert_cmd = "$mertdir/mert"; +my $mert_pro_cmd = "$mertdir/pro"; + +die "Not executable: $mert_extract_cmd" if ! -x $mert_extract_cmd; +die "Not executable: $mert_mert_cmd" if ! -x $mert_mert_cmd; +die "Not executable: $mert_pro_cmd" if ! -x $mert_pro_cmd; + +my $pro_optimizer = "$mertdir/megam_i686.opt"; # or set to your installation +if (($___PAIRWISE_RANKED_OPTIMIZER || $___PRO_STARTING_POINT) && ! -x $pro_optimizer) { + print "did not find $pro_optimizer, installing it in $mertdir\n"; + `cd $mertdir; wget http://www.cs.utah.edu/~hal/megam/megam_i686.opt.gz;`; + `gunzip $pro_optimizer.gz`; + `chmod +x $pro_optimizer`; + die("ERROR: Installation of megam_i686.opt failed! Install by hand from http://www.cs.utah.edu/~hal/megam/") unless -x $pro_optimizer; +} + +$mertargs = "" if !defined $mertargs; + +my $scconfig = undef; +if ($mertargs =~ /\-\-scconfig\s+(.+?)(\s|$)/){ + $scconfig=$1; + $scconfig =~ s/\,/ /g; + $mertargs =~ s/\-\-scconfig\s+(.+?)(\s|$)//; +} + +# handling reference lengh strategy +if (($___CLOSEST + $___AVERAGE + $___SHORTEST) > 1){ + die "You can specify just ONE reference length strategy (closest or shortest or average) not both\n"; +} + +if ($___SHORTEST){ + $scconfig .= " reflen:shortest"; +}elsif ($___AVERAGE){ + $scconfig .= " reflen:average"; +}elsif ($___CLOSEST){ + $scconfig .= " reflen:closest"; +} + +# handling case-insensitive flag +if ($___NOCASE) { + $scconfig .= " case:false"; +}else{ + $scconfig .= " case:true"; +} +$scconfig =~ s/^\s+//; +$scconfig =~ s/\s+$//; +$scconfig =~ s/\s+/,/g; + +$scconfig = "--scconfig $scconfig" if ($scconfig); + +my $mert_extract_args=$mertargs; +$mert_extract_args .=" $scconfig"; +$mert_extract_args .=" $extractorargs"; + +$mertmertargs = "" if !defined $mertmertargs; + +my $mert_mert_args="$mertargs $mertmertargs"; +$mert_mert_args =~ s/\-+(binary|b)\b//; +$mert_mert_args .=" $scconfig"; +if ($___ACTIVATE_FEATURES){ $mert_mert_args .=" -o \"$___ACTIVATE_FEATURES\""; } + +my ($just_cmd_filtercmd,$x) = split(/ /,$filtercmd); +die "Not executable: $just_cmd_filtercmd" if ! -x $just_cmd_filtercmd; +die "Not executable: $moses_parallel_cmd" if defined $___JOBS && ! -x $moses_parallel_cmd; +die "Not executable: $qsubwrapper" if defined $___JOBS && ! -x $qsubwrapper; +# die "Not executable: $___DECODER" if ! -x $___DECODER; + +my $input_abs = ensure_full_path($___DEV_F); +die "File not found: $___DEV_F (interpreted as $input_abs)." +if ! -e $input_abs; + $___DEV_F = $input_abs; + +# Option to pass to qsubwrapper and moses-parallel +my $pass_old_sge = $old_sge ? "-old-sge" : ""; + +my $decoder_abs = ensure_full_path($___DECODER); +die "File not executable: $___DECODER (interpreted as $decoder_abs)." + if ! -x $decoder_abs; +$___DECODER = $decoder_abs; + + +my $config_abs = ensure_full_path($___CONFIG); +die "File not found: $___CONFIG (interpreted as $config_abs)." + if ! -e $config_abs; +$___CONFIG = $config_abs; + +# moses should use our config +if ($___DECODER_FLAGS =~ /(^|\s)-(config|f) / +|| $___DECODER_FLAGS =~ /(^|\s)-(ttable-file|t) / +|| $___DECODER_FLAGS =~ /(^|\s)-(distortion-file) / +|| $___DECODER_FLAGS =~ /(^|\s)-(generation-file) / +|| $___DECODER_FLAGS =~ /(^|\s)-(lmodel-file) / +|| $___DECODER_FLAGS =~ /(^|\s)-(global-lexical-file) / +) { + die "It is forbidden to supply any of -config, -ttable-file, -distortion-file, -generation-file or -lmodel-file in the --decoder-flags.\nPlease use only the --config option to give the config file that lists all the supplementary files."; +} + +# as weights are normalized in the next steps (by cmert) +# normalize initial LAMBDAs, too +my $need_to_normalize = 0; + +#store current directory and create the working directory (if needed) +my $cwd = `pawd 2>/dev/null`; +if(!$cwd){$cwd = `pwd`;} +chomp($cwd); + +$___WORKING_DIR = $cwd if (!defined $___WORKING_DIR); +chomp $___WORKING_DIR; + +# mkpath($___WORKING_DIR); + +{ +# open local scope + +#chdir to the working directory +chdir($___WORKING_DIR) or die "Can't chdir to $___WORKING_DIR"; + +# fixed file names +my $mert_outfile = "mert.out"; +my $mert_logfile = "mert.log"; +my $weights_in_file = "init.opt"; +my $weights_out_file = "weights.txt"; + +# set start run +my $start_run = 1; +my $bestpoint = undef; +my $devbleu = undef; +my $sparse_weights_file = undef; +my $jobid = -1; + +my $prev_feature_file = undef; +my $prev_score_file = undef; +my $prev_init_file = undef; + + +######################### +# set jobid to trace different jobs +my $prevjid = undef; + + + +### load featlist when needed######## +print STDERR "loading feat config ./run$run.moses.ini\n"; +my $featlist = get_featlist_from_moses("./run$run.moses.ini"); +$featlist = insert_ranges_to_featlist($featlist, $___RANGES); + +# Mark which features are disabled: +if (defined $___ACTIVATE_FEATURES) { + my %enabled = map { ($_, 1) } split /[, ]+/, $___ACTIVATE_FEATURES; + my %cnt; + for(my $i=0; $i{"names"}}); $i++) { + my $name = $featlist->{"names"}->[$i]; + $cnt{$name} = 0 if !defined $cnt{$name}; + $featlist->{"enabled"}->[$i] = $enabled{$name."_".$cnt{$name}}; + $cnt{$name}++; + } +} else { + # all enabled + for(my $i=0; $i{"names"}}); $i++) { + $featlist->{"enabled"}->[$i] = 1; + } +} + +##################################### + + + +my $oldallsorted = undef; +my $allsorted = undef; + +my $nbest_file=undef; +my $lsamp_file=undef; #Lattice samples +my $orig_nbest_file=undef; # replaced if lattice sampling +my $cmd=undef; + + + + + my $base_feature_file = "features.dat"; + my $base_score_file = "scores.dat"; + my $feature_file = "run$run.${base_feature_file}"; + my $score_file = "run$run.${base_score_file}"; + + + + # Create the initial weights file for mert: init.opt + foreach (@{$featlist->{"names"}}) { + print STDERR "feature list name: $_\n"; + } + + my @MIN = @{$featlist->{"mins"}}; + my @MAX = @{$featlist->{"maxs"}}; + my @CURR = @{$featlist->{"values"}}; + my @NAME = @{$featlist->{"names"}}; + + open(OUT,"> $weights_in_file") + or die "Can't write $weights_in_file (WD now $___WORKING_DIR)"; + print OUT join(" ", @CURR)."\n"; + print OUT join(" ", @MIN)."\n"; # this is where we could pass MINS + print OUT join(" ", @MAX)."\n"; # this is where we could pass MAXS + close(OUT); + # print join(" ", @NAME)."\n"; + + print "CURR before MERT @CURR\n"; + + # make a backup copy labelled with this run number + safesystem("\\cp -f $weights_in_file run$run.$weights_in_file") or die; + + my $DIM = scalar(@CURR); # number of lambdas + + # run mert + $cmd = "$mert_mert_cmd -d $DIM $mert_mert_args"; + + my $mert_settings = " -n $___RANDOM_RESTARTS"; + my $seed_settings = ""; + if ($___PREDICTABLE_SEEDS) { + my $seed = $run * 1000; + $seed_settings .= " -r $seed"; + } + $mert_settings .= $seed_settings; + if ($___RANDOM_DIRECTIONS) { + if ($___NUM_RANDOM_DIRECTIONS == 0) { + $mert_settings .= " -m 50"; + } + $mert_settings .= " -t random-direction"; + } + if ($___NUM_RANDOM_DIRECTIONS) { + $mert_settings .= " -m $___NUM_RANDOM_DIRECTIONS"; + } + if ($__THREADS) { + $mert_settings .= " --threads $__THREADS"; + } + + ############################## + # construct prev_feature_file and prev_socre_file + + my $firstrun; + if ($prev_aggregate_nbl_size==-1){ + $firstrun=1; + } + else{ + $firstrun=$run-$prev_aggregate_nbl_size+1; + $firstrun=($firstrun>0)?$firstrun:1; + } + ############################## + + print "loading data from $firstrun to $run (prev_aggregate_nbl_size=$prev_aggregate_nbl_size)\n"; + $prev_feature_file = undef; + $prev_score_file = undef; + $prev_init_file = undef; + for (my $i=$firstrun;$i<$run;$i++){ + if (defined $prev_feature_file){ + $prev_feature_file = "${prev_feature_file},run${i}.${base_feature_file}"; + } + else{ + $prev_feature_file = "run${i}.${base_feature_file}"; + } + if (defined $prev_score_file){ + $prev_score_file = "${prev_score_file},run${i}.${base_score_file}"; + } + else{ + $prev_score_file = "run${i}.${base_score_file}"; + } + } + + print STDERR "prev_feature_file is $prev_feature_file\n"; + print STDERR "prev_score_files is $prev_score_file\n"; + print STDERR "prev_init_files is undefined\n"; + ############################ + + + my $ffiles = ""; + my $scfiles = ""; + if (defined $prev_feature_file) { + $ffiles = "$prev_feature_file,$feature_file"; + } + else{ + $ffiles = "$feature_file"; + } + if (defined $prev_score_file) { + $scfiles = "$prev_score_file,$score_file"; + } + else{ + $scfiles = "$score_file"; + } + + my $file_settings = " --ffile $ffiles --scfile $scfiles"; + my $pro_file_settings = "--ffile " . join( " --ffile ", split(/,/, $ffiles)) . + " --scfile " . join( " --scfile ", split(/,/, $scfiles)); + + if ($___START_WITH_HISTORIC_BESTS && defined $prev_init_file) { + $file_settings .= " --ifile $prev_init_file,run$run.$weights_in_file"; + } + else{ + $file_settings .= " --ifile run$run.$weights_in_file"; + } + + $cmd .= $file_settings; + + # pro optimization + if ($___PAIRWISE_RANKED_OPTIMIZER) { + print STDERR "\$___PAIRWISE_RANKED_OPTIMIZER not supported in non-sync mode\n"; + exit 1; + } + # first pro, then mert + elsif ($___PRO_STARTING_POINT) { + print STDERR "\$___PRO_STARTING_POINT not supported in non-sync mode\n"; + exit 1; + # run pro... + my $pro_cmd = "$mert_pro_cmd $seed_settings $pro_file_settings -o run$run.pro.data ; $pro_optimizer -fvals -maxi 30 -nobias binary run$run.pro.data"; + &submit_or_exec($pro_cmd,"run$run.pro.out","run$run.pro.err"); + # ... get results ... + my %dummy; + ($bestpoint,$devbleu) = &get_weights_from_mert("run$run.pro.out","run$run.pro.err",scalar @{$featlist->{"names"}},\%dummy); + open(PRO_START,">run$run.init.pro"); + print PRO_START $bestpoint."\n"; + close(PRO_START); + # ... and run mert + $cmd =~ s/(--ifile \S+)/$1,run$run.init.pro/; + &submit_or_exec($cmd.$mert_settings,$mert_outfile,$mert_logfile); + } + # just mert + else { + print STDERR "I am running just MERT\n"; + &submit_or_exec($cmd.$mert_settings,$mert_outfile,$mert_logfile); + } + + die "Optimization failed, file $weights_out_file does not exist or is empty" + if ! -s $weights_out_file; + + # backup copies + safesystem ("\\cp -f extract.err run$run.extract.err") or die; + safesystem ("\\cp -f extract.out run$run.extract.out") or die; + safesystem ("\\cp -f $mert_outfile run$run.$mert_outfile") or die; + safesystem ("\\cp -f $mert_logfile run$run.$mert_logfile") or die; + safesystem ("touch $mert_logfile run$run.$mert_logfile") or die; + safesystem ("\\cp -f $weights_out_file run$run.$weights_out_file") or die; # this one is needed for restarts, too + + print "run $run end at ".`date`; + + my %sparse_weights; # sparse features + ($bestpoint,$devbleu) = &get_weights_from_mert("run$run.$mert_outfile","run$run.$mert_logfile",scalar @{$featlist->{"names"}},\%sparse_weights); + + die "Failed to parse mert.log, missed Best point there." + if !defined $bestpoint || !defined $devbleu; + + print "($run) BEST at $run: $bestpoint => $devbleu at ".`date`; + + # update my cache of lambda values + my @newweights = split /\s+/, $bestpoint; + + # interpolate with prior's interation weight, if historic-interpolation is specified + if ($___HISTORIC_INTERPOLATION>0 && $run>3) { + my %historic_sparse_weights; + if (-e "run$run.sparse-weights") { + open(SPARSE,"run$run.sparse-weights"); + while() { + chop; + my ($feature,$weight) = split; + $historic_sparse_weights{$feature} = $weight; + } + } + my $prev = $run-1; + my @historic_weights = split /\s+/, `cat run$prev.$weights_out_file`; + for(my $i=0;$i $sparse_weights{$_}\n"; + } + foreach (keys %historic_sparse_weights) { + $sparse_weights{$_} += (1-$___HISTORIC_INTERPOLATION) * $historic_sparse_weights{$_}; + #print STDERR "sparse_weights{$_} += (1-$___HISTORIC_INTERPOLATION) * $historic_sparse_weights{$_} -> $sparse_weights{$_}\n"; + } + } + if ($___HISTORIC_INTERPOLATION>0) { + open(WEIGHTS,">run$run.$weights_out_file"); + print WEIGHTS join(" ",@newweights); + close(WEIGHTS); + } + + $featlist->{"values"} = \@newweights; + + if (scalar keys %sparse_weights) { + $sparse_weights_file = "run".($run+1).".sparse-weights"; + open(SPARSE,">".$sparse_weights_file); + foreach my $feature (keys %sparse_weights) { + print SPARSE "$feature $sparse_weights{$feature}\n"; + } + close(SPARSE); + } + + ## additional stopping criterion: weights have not changed + my $shouldstop = 1; + print "CURR after MERT: @CURR\n"; + + for(my $i=0; $i<@CURR; $i++) { + die "Lost weight! mert reported fewer weights (@newweights) than we gave it (@CURR)" + if !defined $newweights[$i]; + # print STDERR "$i Current weight : $CURR[$i]\n"; + # print STDERR "$i New weight: $newweights[$i]\n"; + if (abs($CURR[$i] - $newweights[$i]) >= $minimum_required_change_in_weights) { + $shouldstop = 0; + # last; + } + } + + + open F, "> finished_step.txt" or die "Can't mark finished step"; + print F $run."\n"; + close F; + + ##################################################33 + print "loading data from $prev_feature_file\n" if defined($prev_feature_file); + print "loading data from $prev_score_file\n" if defined($prev_score_file); + print "loading data from $prev_init_file\n" if defined($prev_init_file); + + + ### extra step to create config for next step ################### + $nextrun=$run+1; + + create_config($___CONFIG, "./run$nextrun.moses.ini", $featlist, $nextrun, $devbleu, $sparse_weights_file); + ################################################################# + + + if ($shouldstop) { + print STDERR "None of the weights changed more than $minimum_required_change_in_weights. Stopping.\n"; + + for (my $i=$run+1;$i<=$maximum_iterations;$i++){ + kill_unnecessary_jobs($submithost,$i); + } + last; ## end while loop + } + + + + +} # end of local scope + + + +sub kill_unnecessary_jobs { + my ($submithost,$iter_id) = @_; + my @sgepids = (); + my @processids = (); + my $jobid = ""; + my $pidid = undef; + my $kill_target = ""; + my $ssh = undef; + + ## read all relevant jobid ######### + chomp($jobid=`cat decode$iter_id.id decode$iter_id.POLL.id zipext$iter_id.id processmoses$iter_id.id`); + + if ($jobid != "") { + @sgepids = split("\n",$jobid); + $kill_target = join(" ",@sgepids); + } + #################################### + + if ($kill_target != "") { + chomp(my $my_username = `whoami`); + $ssh = Net::OpenSSH::Compat::Perl->new($submithost, debug=>0); + + $ssh->login("$my_username",`cat /home/$my_username/accpw`); + $ssh->cmd("qdel $kill_target"); + } + + + + ## read relevant pid ############### + chomp($pidid=`ls decode$iter_id.id.pid decode$iter_id.POLL.id.pid zipext$iter_id.id.pid processmoses$iter_id.id.pid`); + + if (defined $pidid) { + @processids = split("\n",$pidid); + } + ################################### + + + foreach my $kill_pidfile (@processids) { + print STDERR "Force delete process represented by $kill_pidfile\n"; + &force_exit_submit_thu_host($submithost,$kill_pidfile); + } + #################################### + + + + +} + +sub get_weights_from_mert { + my ($outfile,$logfile,$weight_count,$sparse_weights) = @_; + my ($bestpoint,$devbleu); + + if ($___PAIRWISE_RANKED_OPTIMIZER || ($___PRO_STARTING_POINT && $logfile =~ /pro/)) { + open(IN,$outfile) or die "Can't open $outfile"; + my (@WEIGHT,$sum); + for(my $i=0;$i<$weight_count;$i++) { push @WEIGHT, 0; } + while() { + # regular features + if (/^F(\d+) ([\-\.\de]+)/) { + $WEIGHT[$1] = $2; + $sum += abs($2); + } + # sparse features + elsif(/^(.+_.+) ([\-\.\de]+)/) { + $$sparse_weights{$1} = $2; + } + } + $devbleu = "unknown"; + foreach (@WEIGHT) { $_ /= $sum; } + foreach (keys %{$sparse_weights}) { $$sparse_weights{$_} /= $sum; } + $bestpoint = join(" ",@WEIGHT); + close IN; + } + else { + open(IN,$logfile) or die "Can't open $logfile"; + while () { + if (/Best point:\s*([\s\d\.\-e]+?)\s*=> ([\-\d\.]+)/) { + $bestpoint = $1; + $devbleu = $2; + last; + } + } + close IN; + } + return ($bestpoint,$devbleu); +} + + +sub insert_ranges_to_featlist { + my $featlist = shift; + my $ranges = shift; + + $ranges = [] if !defined $ranges; + + # first collect the ranges from options + my $niceranges; + foreach my $range (@$ranges) { + my $name = undef; + foreach my $namedpair (split /,/, $range) { + if ($namedpair =~ /^(.*?):/) { + $name = $1; + $namedpair =~ s/^.*?://; + die "Unrecognized name '$name' in --range=$range" + if !defined $ABBR2FULL{$name}; + } + my ($min, $max) = split /\.\./, $namedpair; + die "Bad min '$min' in --range=$range" if $min !~ /^-?[0-9.]+$/; + die "Bad max '$max' in --range=$range" if $min !~ /^-?[0-9.]+$/; + die "No name given in --range=$range" if !defined $name; + push @{$niceranges->{$name}}, [$min, $max]; + } + } + + # now populate featlist + my $seen = undef; + for(my $i=0; $i{"names"}}); $i++) { + my $name = $featlist->{"names"}->[$i]; + $seen->{$name} ++; + my $min = 0.0; + my $max = 1.0; + if (defined $niceranges->{$name}) { + my $minmax = shift @{$niceranges->{$name}}; + ($min, $max) = @$minmax if defined $minmax; + } + $featlist->{"mins"}->[$i] = $min; + $featlist->{"maxs"}->[$i] = $max; + } + return $featlist; + } + +sub sanity_check_order_of_lambdas { + my $featlist = shift; + my $filename_or_stream = shift; + + my @expected_lambdas = @{$featlist->{"names"}}; + my @got = get_order_of_scores_from_nbestlist($filename_or_stream); + die "Mismatched lambdas. Decoder returned @got, we expected @expected_lambdas" + if "@got" ne "@expected_lambdas"; +} + +sub get_featlist_from_moses { + # run moses with the given config file and return the list of features and + # their initial values + + my $configfn = shift; + my $featlistfn = "./features.list.run${run}_end"; + if (-e $featlistfn) { + print STDERR "Deleting featlistfn: $featlistfn\n"; + } + + print STDERR "Asking moses for feature names and values from $___CONFIG\n"; + my $cmd = "$___DECODER $___DECODER_FLAGS -config $configfn -inputtype $___INPUTTYPE -show-weights > $featlistfn"; + safesystem($cmd) or die "Failed to run moses with the config $configfn"; + + + + + # read feature list + my @names = (); + my @startvalues = (); + open(INI,$featlistfn) or die "Can't read $featlistfn"; + my $nr = 0; + my @errs = (); + while () { + $nr++; + chomp; + /^(.+) (\S+) (\S+)$/ || die("invalid feature: $_"); + my ($longname, $feature, $value) = ($1,$2,$3); + next if $value eq "sparse"; + push @errs, "$featlistfn:$nr:Bad initial value of $feature: $value\n" + if $value !~ /^[+-]?[0-9.e]+$/; + push @errs, "$featlistfn:$nr:Unknown feature '$feature', please add it to \@ABBR_FULL_MAP\n" + if !defined $ABBR2FULL{$feature}; + push @names, $feature; + push @startvalues, $value; + } + close INI; + if (scalar @errs) { + print STDERR join("", @errs); + exit 1; + } + return {"names"=>\@names, "values"=>\@startvalues}; +} + +sub get_order_of_scores_from_nbestlist { + # read the first line and interpret the ||| label: num num num label2: num ||| column in nbestlist + # return the score labels in order + my $fname_or_source = shift; + # print STDERR "Peeking at the beginning of nbestlist to get order of scores: $fname_or_source\n"; + open IN, $fname_or_source or die "Failed to get order of scores from nbestlist '$fname_or_source'"; + my $line = ; + close IN; + die "Line empty in nbestlist '$fname_or_source'" if !defined $line; + my ($sent, $hypo, $scores, $total) = split /\|\|\|/, $line; + $scores =~ s/^\s*|\s*$//g; + die "No scores in line: $line" if $scores eq ""; + + my @order = (); + my $label = undef; + my $sparse = 0; # we ignore sparse features here + foreach my $tok (split /\s+/, $scores) { + if ($tok =~ /.+_.+:/) { + $sparse = 1; + } elsif ($tok =~ /^([a-z][0-9a-z]*):/i) { + $label = $1; + } elsif ($tok =~ /^-?[-0-9.e]+$/) { + if (!$sparse) { + # a score found, remember it + die "Found a score but no label before it! Bad nbestlist '$fname_or_source'!" + if !defined $label; + push @order, $label; + } + $sparse = 0; + } else { + die "Not a label, not a score '$tok'. Failed to parse the scores string: '$scores' of nbestlist '$fname_or_source'"; + } + } + print STDERR "The decoder returns the scores in this order: @order\n"; + return @order; +} + + +sub safesystem { + print STDERR "Executing: @_\n"; + system(@_); + if ($? == -1) { + print STDERR "Failed to execute: @_\n $!\n"; + exit(1); + } + elsif ($? & 127) { + printf STDERR "Execution of: @_\n died with signal %d, %s coredump\n", + ($? & 127), ($? & 128) ? 'with' : 'without'; + exit(1); + } + else { + my $exitcode = $? >> 8; + print STDERR "Exit code: $exitcode\n" if $exitcode; + return ! $exitcode; + } +} +sub ensure_full_path { + my $PATH = shift; +$PATH =~ s/\/nfsmnt//; + return $PATH if $PATH =~ /^\//; + my $dir = `pawd 2>/dev/null`; + if(!$dir){$dir = `pwd`;} + chomp($dir); + $PATH = $dir."/".$PATH; + $PATH =~ s/[\r\n]//g; + $PATH =~ s/\/\.\//\//g; + $PATH =~ s/\/+/\//g; + my $sanity = 0; + while($PATH =~ /\/\.\.\// && $sanity++<10) { + $PATH =~ s/\/+/\//g; + $PATH =~ s/\/[^\/]+\/\.\.\//\//g; + } + $PATH =~ s/\/[^\/]+\/\.\.$//; + $PATH =~ s/\/+$//; +$PATH =~ s/\/nfsmnt//; + return $PATH; +} + +sub submit_or_exec { + + my $argvlen = @_; + my $cmd = undef; + my $stdout = undef; + my $stderr = undef; + my $jidfile = undef; + my $prevjid = undef; + + # if supply 3 arguments, exec without submit + # if supply 4 arguments, then submit new job + # if supply 5 arguments, wait for the previous job to finish + if ($argvlen == 3){ + ($cmd,$stdout,$stderr) = @_; + } elsif ($argvlen == 4){ + ($cmd,$stdout,$stderr,$jidfile) = @_; + } elsif ($argvlen == 5){ + ($cmd,$stdout,$stderr,$jidfile,$prevjid) = @_; + } + + print STDERR "exec: $cmd\n"; + if (defined $___JOBS && $___JOBS > 0 && $argvlen==5) { + safesystem("$qsubwrapper $pass_old_sge -command='$cmd' -queue-parameters=\"$queue_flags\" -stdout=$stdout -stderr=$stderr -jidfile=$jidfile -prevjid=$prevjid" ) + or die "ERROR: Failed to submit '$cmd' (via $qsubwrapper)"; + } + elsif (defined $___JOBS && $___JOBS > 0 && $argvlen==4) { + safesystem("$qsubwrapper $pass_old_sge -command='$cmd' -queue-parameters=\"$queue_flags\" -stdout=$stdout -stderr=$stderr -jidfile=$jidfile" ) + or die "ERROR: Failed to submit '$cmd' (via $qsubwrapper)"; + } else { + safesystem("$cmd > $stdout 2> $stderr") or die "ERROR: Failed to run '$cmd'."; + } +} + +sub exit_submit { + + my $argvlen = @_; + my $cmd = undef; + my $stdout = undef; + my $stderr = undef; + my $jidfile = undef; + my $pidfile = undef; + my $prevjid = undef; + my $prevjidarraysize = 0; + my @prevjidarray = (); + my $pid = undef; + my $qsubcmd=""; + my $hj=""; + + # if supply 4 arguments, then submit new job + # if supply 5 arguments, wait for the previous job to finish + if ($argvlen == 2) { + ($stdout,$stderr) = @_; + } elsif ($argvlen == 4){ + ($stdout,$stderr,$jidfile,$pidfile) = @_; + } elsif ($argvlen == 5){ + ($stdout,$stderr,$jidfile,$pidfile,$prevjid) = @_; + } + + # parse prevjid ######################## + $prevjid =~ s/^\s+|\s+$//g; + @prevjidarray = split(/\s+/,$prevjid); + $prevjidarraysize = scalar(@prevjidarray); + ######################################## + + + # print STDERR "exec: $stdout\n"; + + # read pid from file, and draft exit script ################## + chomp ($pid=`tail -n 1 $pidfile`); + open (OUT, ">exitjob$pid.sh"); + + my $scriptheader="\#\!/bin/bash\n\#\$ -S /bin/sh\n# Both lines are needed to invoke base\n#the above line is ignored by qsub, unless parameter \"-b yes\" is set!\n\n"; + $scriptheader .="uname -a\n\n"; + $scriptheader .="cd $___WORKING_DIR\n\n"; + + print OUT $scriptheader; + + print OUT "if $qsubwrapper_exit -submithost=$submithost -stdout=$stdout -stderr=$stderr -jidfile=$jidfile -pidfile=$pidfile > exitjob$pid.out 2> exitjob$pid.err ; then + echo 'succeeded' +else + echo failed with exit status \$\? + die=1 +fi +"; + print OUT "\n\n"; + + close (OUT); + # setting permissions of the script + chmod(oct(755),"exitjob$pid.sh"); + ############################################################## + + + if (defined $___JOBS && $___JOBS > 0 && $argvlen==5) { + if (defined $prevjid && $prevjid!=-1 && $prevjidarraysize == 1){ + $hj = "-hold_jid $prevjid"; + } elsif (defined $prevjid && $prevjidarraysize > 1){ + $hj = "-hold_jid " . join(" -hold_jid ", @prevjidarray); + } + $qsubcmd="qsub $queue_flags -V $hj exitjob$pid.sh > exitjob$pid.log 2>&1"; + safesystem($qsubcmd) or die "ERROR: Failed to exit-submit $pid (via $qsubwrapper_exit)"; + } elsif (defined $___JOBS && $___JOBS > 0 && $argvlen==4) { + $qsubcmd="qsub $queue_flags -V exitjob$pid.sh > exitjob$pid.log 2>&1"; + safesystem($qsubcmd) or die "ERROR: Failed to exit-submit $pid (via $qsubwrapper_exit)"; + } else { + safesystem("rm $stdout") or die "ERROR: Failed to remove '$stdout'."; + safesystem("rm $stderr") or die "ERROR: Failed to remove '$stderr'."; + } +} + + + + + + + +sub create_extractor_script() +{ + my ($cmd, $outdir) = @_; + my $script_path = File::Spec->catfile($outdir, "extractor.sh"); + + open my $out, '>', $script_path + or die "Couldn't open $script_path for writing: $!\n"; + print $out "#!/bin/bash\n"; + print $out "cd $outdir\n"; + print $out "$cmd\n"; + close($out); + + `chmod +x $script_path`; + + return $script_path; +} + + +sub create_config { + my $infn = shift; # source config + my $outfn = shift; # where to save the config + my $featlist = shift; # the lambdas we should write + my $iteration = shift; # just for verbosity + my $bleu_achieved = shift; # just for verbosity + my $sparse_weights_file = shift; # only defined when optimizing sparse features + + my %P; # the hash of all parameters we wish to override + + # first convert the command line parameters to the hash + { # ensure local scope of vars + my $parameter=undef; + print "Parsing --decoder-flags: |$___DECODER_FLAGS|\n"; + $___DECODER_FLAGS =~ s/^\s*|\s*$//; + $___DECODER_FLAGS =~ s/\s+/ /; + foreach (split(/ /,$___DECODER_FLAGS)) { + if (/^\-([^\d].*)$/) { + $parameter = $1; + $parameter = $ABBR2FULL{$parameter} if defined($ABBR2FULL{$parameter}); + } + else { + die "Found value with no -paramname before it: $_" + if !defined $parameter; + push @{$P{$parameter}},$_; + } + } + } + + # First delete all weights params from the input, we're overwriting them. + # Delete both short and long-named version. + for(my $i=0; $i{"names"}}); $i++) { + my $name = $featlist->{"names"}->[$i]; + delete($P{$name}); + delete($P{$ABBR2FULL{$name}}); + } + + # Convert weights to elements in P + for(my $i=0; $i{"names"}}); $i++) { + my $name = $featlist->{"names"}->[$i]; + my $val = $featlist->{"values"}->[$i]; + $name = defined $ABBR2FULL{$name} ? $ABBR2FULL{$name} : $name; + # ensure long name + push @{$P{$name}}, $val; + } + + if (defined($sparse_weights_file)) { + push @{$P{"weights-file"}}, $___WORKING_DIR."/".$sparse_weights_file; + } + + # create new moses.ini decoder config file by cloning and overriding the original one + open(INI,$infn) or die "Can't read $infn"; + delete($P{"config"}); # never output + print "Saving new config to: $outfn\n"; + open(OUT,"> $outfn") or die "Can't write $outfn"; + print OUT "# MERT optimized configuration\n"; + print OUT "# decoder $___DECODER\n"; + print OUT "# BLEU $bleu_achieved on dev $___DEV_F\n"; + print OUT "# We were before running iteration $iteration\n"; + print OUT "# finished ".`date`; + my $line = ; + while(1) { + last unless $line; + + # skip until hit [parameter] + if ($line !~ /^\[(.+)\]\s*$/) { + $line = ; + print OUT $line if $line =~ /^\#/ || $line =~ /^\s+$/; + next; + } + + # parameter name + my $parameter = $1; + $parameter = $ABBR2FULL{$parameter} if defined($ABBR2FULL{$parameter}); + print OUT "[$parameter]\n"; + + # change parameter, if new values + if (defined($P{$parameter})) { + # write new values + foreach (@{$P{$parameter}}) { + print OUT $_."\n"; + } + delete($P{$parameter}); + # skip until new parameter, only write comments + while($line = ) { + print OUT $line if $line =~ /^\#/ || $line =~ /^\s+$/; + last if $line =~ /^\[/; + last unless $line; + } + next; + } + # unchanged parameter, write old + while($line = ) { + last if $line =~ /^\[/; + print OUT $line; + } + } + + # write all additional parameters + foreach my $parameter (keys %P) { + print OUT "\n[$parameter]\n"; + foreach (@{$P{$parameter}}) { + print OUT $_."\n"; + } + } + + close(INI); + close(OUT); + print STDERR "Saved: $outfn\n"; +} + + + + +sub force_exit_submit_thu_host { + + my $argvlen = @_; + my $submithost = undef; + my $run = -1; + my $idx = ""; + my $batch_and_join = ""; + my $my_username = undef; + my $cmd = undef; + my $stdout = undef; + my $stderr = undef; + my $jidfile = undef; + my $pidfile = undef; + my $prevjid = undef; + my $prevjidarraysize = 0; + my @prevjidarray = (); + my $pid = undef; + my $qsubcmd=""; + my $hj=""; + + # if supply 8 arguments, then submit new job + # if supply 9 arguments, wait for the previous job to finish + ($submithost,$pidfile) = @_; + + #if ($argvlen == 6){ + # ($submithost,$run,$idx,$batch_and_join,$stdout,$stderr) = @_; + #} elsif ($argvlen == 8){ + # ($submithost,$run,$idx,$batch_and_join,$stdout,$stderr,$jidfile,$pidfile) = @_; + #} elsif ($argvlen == 9){ + # ($submithost,$run,$idx,$batch_and_join,$stdout,$stderr,$jidfile,$pidfile,$prevjid) = @_; + #} + + # parse prevjid ######################## + #$prevjid =~ s/^\s+|\s+$//g; + #@prevjidarray = split(/\s+/,$prevjid); + $prevjidarraysize = scalar(@prevjidarray); + ######################################## + + # print STDERR "exec: $stdout\n"; + + # read pid from file, and draft exit script ################## + chomp ($pid=`tail -n 1 $pidfile`); + open (OUT, ">forceexitjob$pid.sh"); + + my $scriptheader="\#\!/bin/bash\n\#\$ -S /bin/sh\n# Both lines are needed to invoke base\n#the above line is ignored by qsub, unless parameter \"-b yes\" is set!\n\n"; + $scriptheader .="uname -a\n\n"; + $scriptheader .="cd $___WORKING_DIR\n\n"; + + print OUT $scriptheader; + + print OUT "if $qsubwrapper_exit -submithost=$submithost -force-delete=1 -pidfile=$pidfile > forceexitjob$pid.out 2> forceexitjob$pid.err ; then + echo 'succeeded' +else + echo failed with exit status \$\? + die=1 +fi +"; + print OUT "\n\n"; + + close (OUT); + # setting permissions of the script + chmod(oct(755),"forceexitjob$pid.sh"); + ############################################################## + # + # log in submit host ######################################### + # chomp(my $my_username = `whoami`); + # my $ssh = Net::OpenSSH::Compat::Perl->new($submithost, debug=>0); + + # $ssh->login("$my_username",`cat /home/$my_username/accpw`); + ############################################################## + + + # $qsubcmd="qsub $queue_flags -sync y -o /dev/null -e /dev/null forceexitjob$pid.sh > forceexitjob$pid.log 2>&1"; + # $ssh->cmd("cd $___WORKING_DIR && $qsubcmd"); + safesystem("./forceexitjob$pid.sh"); + + print STDERR "Executing $qsubcmd in $___WORKING_DIR\n"; + +} + + + + diff --git a/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/run-decoder-sge-nosync.pl b/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/run-decoder-sge-nosync.pl new file mode 100644 index 0000000000000000000000000000000000000000..ab0c698af0bcb4db414e40f9854afd7c7cdb5d40 --- /dev/null +++ b/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/run-decoder-sge-nosync.pl @@ -0,0 +1,271 @@ +#!/usr/bin/perl -w + +# $Id$ +# after filter-mode-given-input.pl, process the feature list + +# original code by Philipp Koehn +# changes by Ondrej Bojar +# adapted for hierarchical models by Phil Williams + +use strict; + +use FindBin qw($Bin); +use Getopt::Long; + + + +my $SCRIPTS_ROOTDIR; +if (defined($ENV{"SCRIPTS_ROOTDIR"})) { + $SCRIPTS_ROOTDIR = $ENV{"SCRIPTS_ROOTDIR"}; +} else { + $SCRIPTS_ROOTDIR = $Bin; + if ($SCRIPTS_ROOTDIR eq '') { + $SCRIPTS_ROOTDIR = dirname(__FILE__); + } + $SCRIPTS_ROOTDIR =~ s/\/training$//; + $ENV{"SCRIPTS_ROOTDIR"} = $SCRIPTS_ROOTDIR; +} + + +##!# # moses.ini file uses FULL names for lambdas, while this training script +##!# # internally (and on the command line) uses ABBR names. +##!# my @ABBR_FULL_MAP = qw(d=weight-d lm=weight-l tm=weight-t w=weight-w +##!# g=weight-generation lex=weight-lex I=weight-i); +##!# my %ABBR2FULL = map {split/=/,$_,2} @ABBR_FULL_MAP; +##!# my %FULL2ABBR = map {my ($a, $b) = split/=/,$_,2; ($b, $a);} @ABBR_FULL_MAP; + + +my $verbose = 0; +my $usage = 0; # request for --help + + + + +##!# # consider phrases in input up to $MAX_LENGTH +##!# # in other words, all phrase-tables will be truncated at least to 10 words per +##!# # phrase. +##!# my $MAX_LENGTH = 10; + +# utilities +##!# my $ZCAT = "gzip -cd"; + +# get optional parameters +##!# my $opt_hierarchical = 0; +##!# my $binarizer = undef; +##!# my $opt_min_non_initial_rule_count = undef; +##!# my $opt_gzip = 1; # gzip output files (so far only phrase-based ttable until someone tests remaining models and formats) + +my $___RANGES = undef; +my $___ACTIVATE_FEATURES = undef; # comma-separated (or blank-separated) list of features to work on + # if undef work on all features + # (others are fixed to the starting values) +my $___DECODER_FLAGS = ""; # additional parametrs to pass to the decoder + +# set 0 if input type is text, set 1 if input type is confusion network +my $___INPUTTYPE = 0; + +my $___DECODER = undef; # required, pathname to the decoder executable +my $___CONFIG = undef; # required, pathname to startup ini file + + +GetOptions( +##!# "activate-features=s" => \$___ACTIVATE_FEATURES, #comma-separated (or blank-separated) list of features to work on (others are fixed to the starting values) +##!# "range=s@" => \$___RANGES, +##!# "decoder-flags=s" => \$___DECODER_FLAGS, +##!# "inputtype=i" => \$___INPUTTYPE + "" +) or exit(1); + + + +# the ?? required parameters can be supplied on the command line directly +# or using the --options +if (scalar @ARGV == 2) { + # required parameters: options + $___DECODER = shift; + $___CONFIG = shift; +} + +if ($usage || !defined $___DECODER || !defined $___CONFIG) { + print STDERR "usage: $0 \$___DECODER \$___CONFIG(decoder.ini) +Options: + --activate-features=STRING ... comma-separated list of features to optimize, + others are fixed to the starting values + default: optimize all features + example: tm_0,tm_4,d_0 + --range=tm:0..1,-1..1 ... specify min and max value for some features + --range can be repeated as needed. + The order of the various --range specifications + is important only within a feature name. + E.g.: + --range=tm:0..1,-1..1 --range=tm:0..2 + is identical to: + --range=tm:0..1,-1..1,0..2 + but not to: + --range=tm:0..2 --range=tm:0..1,-1..1 + --decoder-flags=STRING ... extra parameters for the decoder + --inputtype=[0|1|2] ... Handle different input types: (0 for text, + 1 for confusion network, 2 for lattices, + default is 0) +"; + exit 1; +} + + + + +############################################################ +############################################################ +############################################################ + +# main + +# we run moses to check validity of moses.ini and to obtain all the feature +# names +my $featlist = get_featlist_from_moses($___CONFIG); +$featlist = insert_ranges_to_featlist($featlist, $___RANGES); + + +# Mark which features are disabled: +if (defined $___ACTIVATE_FEATURES) { + my %enabled = map { ($_, 1) } split /[, ]+/, $___ACTIVATE_FEATURES; + my %cnt; + for(my $i=0; $i{"names"}}); $i++) { + my $name = $featlist->{"names"}->[$i]; + $cnt{$name} = 0 if !defined $cnt{$name}; + $featlist->{"enabled"}->[$i] = $enabled{$name."_".$cnt{$name}}; + $cnt{$name}++; + } +} else { + # all enabled + for(my $i=0; $i{"names"}}); $i++) { + $featlist->{"enabled"}->[$i] = 1; + } +} + +print STDERR "MERT starting values and ranges for random generation:\n"; +for(my $i=0; $i{"names"}}); $i++) { + my $name = $featlist->{"names"}->[$i]; + my $val = $featlist->{"values"}->[$i]; + my $min = $featlist->{"mins"}->[$i]; + my $max = $featlist->{"maxs"}->[$i]; + my $enabled = $featlist->{"enabled"}->[$i]; + printf STDERR " %5s = %7.3f", $name, $val; + if ($enabled) { + printf STDERR " (%5.2f .. %5.2f)\n", $min, $max; + } else { + print STDERR " --- inactive, not optimized ---\n"; + } +} + + + + + +sub get_featlist_from_moses { + # run moses with the given config file and return the list of features and + # their initial values + my $configfn = shift; + my $featlistfn = "./features.list"; + if (-e $featlistfn) { + print STDERR "Using cached features list: $featlistfn\n"; + } else { + print STDERR "Asking moses for feature names and values from $___CONFIG\n"; + my $cmd = "$___DECODER $___DECODER_FLAGS -config $configfn -inputtype $___INPUTTYPE -show-weights > $featlistfn"; + print STDERR "$cmd\n"; #DEBUG + safesystem($cmd) or die "Failed to run moses with the config $configfn"; + } + + # read feature list + my @names = (); + my @startvalues = (); + open(INI,$featlistfn) or die "Can't read $featlistfn"; + my $nr = 0; + my @errs = (); + while () { + $nr++; + chomp; + /^(.+) (\S+) (\S+)$/ || die("invalid feature: $_"); + my ($longname, $feature, $value) = ($1,$2,$3); + next if $value eq "sparse"; + push @errs, "$featlistfn:$nr:Bad initial value of $feature: $value\n" + if $value !~ /^[+-]?[0-9.e]+$/; + push @errs, "$featlistfn:$nr:Unknown feature '$feature', please add it to \@ABBR_FULL_MAP\n" + if !defined $ABBR2FULL{$feature}; + push @names, $feature; + push @startvalues, $value; + } + close INI; + if (scalar @errs) { + print STDERR join("", @errs); + exit 1; + } + return {"names"=>\@names, "values"=>\@startvalues}; +} + + +sub insert_ranges_to_featlist { + my $featlist = shift; + my $ranges = shift; + + $ranges = [] if !defined $ranges; + + # first collect the ranges from options + my $niceranges; + foreach my $range (@$ranges) { + my $name = undef; + foreach my $namedpair (split /,/, $range) { + if ($namedpair =~ /^(.*?):/) { + $name = $1; + $namedpair =~ s/^.*?://; + die "Unrecognized name '$name' in --range=$range" + if !defined $ABBR2FULL{$name}; + } + my ($min, $max) = split /\.\./, $namedpair; + die "Bad min '$min' in --range=$range" if $min !~ /^-?[0-9.]+$/; + die "Bad max '$max' in --range=$range" if $min !~ /^-?[0-9.]+$/; + die "No name given in --range=$range" if !defined $name; + push @{$niceranges->{$name}}, [$min, $max]; + } + } + + # now populate featlist + my $seen = undef; + for(my $i=0; $i{"names"}}); $i++) { + my $name = $featlist->{"names"}->[$i]; + $seen->{$name} ++; + my $min = 0.0; + my $max = 1.0; + if (defined $niceranges->{$name}) { + my $minmax = shift @{$niceranges->{$name}}; + ($min, $max) = @$minmax if defined $minmax; + } + $featlist->{"mins"}->[$i] = $min; + $featlist->{"maxs"}->[$i] = $max; + } + return $featlist; +} + +sub safesystem { + print STDERR "Executing: @_\n"; + system(@_); + if ($? == -1) { + print STDERR "Failed to execute: @_\n $!\n"; + exit(1); + } + elsif ($? & 127) { + printf STDERR "Execution of: @_\n died with signal %d, %s coredump\n", + ($? & 127), ($? & 128) ? 'with' : 'without'; + exit(1); + } + else { + my $exitcode = $? >> 8; + print STDERR "Exit code: $exitcode\n" if $exitcode; + return ! $exitcode; + } +} + + + + + diff --git a/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/zipextract-decoder-result.pl b/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/zipextract-decoder-result.pl new file mode 100644 index 0000000000000000000000000000000000000000..defc4fda78d1dcc60c2227a289dcf6e538a3ddd4 --- /dev/null +++ b/mosesdecoder/contrib/mert-sge-nosync/training/sge-nosync/zipextract-decoder-result.pl @@ -0,0 +1,832 @@ +#!/usr/bin/perl -w +# $Id$ +# Usage: +# mert-moses.pl +# For other options see below or run 'mert-moses.pl --help' + +# Notes: +# and should be raw text files, one sentence per line +# can be a prefix, in which case the files are 0, 1, etc. are used + +# Excerpts from revision history + +# Sept 2011 multi-threaded mert (Barry Haddow) +# 3 Aug 2011 Added random directions, historic best, pairwise ranked (PK) +# Jul 2011 simplifications (Ondrej Bojar) +# -- rely on moses' -show-weights instead of parsing moses.ini +# ... so moses is also run once *before* mert starts, checking +# the model to some extent +# -- got rid of the 'triples' mess; +# use --range to supply bounds for random starting values: +# --range tm:-3..3 --range lm:-3..3 +# 5 Aug 2009 Handling with different reference length policies (shortest, average, closest) for BLEU +# and case-sensistive/insensitive evaluation (Nicola Bertoldi) +# 5 Jun 2008 Forked previous version to support new mert implementation. +# 13 Feb 2007 Better handling of default values for lambda, now works with multiple +# models and lexicalized reordering +# 11 Oct 2006 Handle different input types through parameter --inputype=[0|1] +# (0 for text, 1 for confusion network, default is 0) (Nicola Bertoldi) +# 10 Oct 2006 Allow skip of filtering of phrase tables (--no-filter-phrase-table) +# useful if binary phrase tables are used (Nicola Bertoldi) +# 28 Aug 2006 Use either closest or average or shortest (default) reference +# length as effective reference length +# Use either normalization or not (default) of texts (Nicola Bertoldi) +# 31 Jul 2006 move gzip run*.out to avoid failure wit restartings +# adding default paths +# 29 Jul 2006 run-filter, score-nbest and mert run on the queue (Nicola; Ondrej had to type it in again) +# 28 Jul 2006 attempt at foolproof usage, strong checking of input validity, merged the parallel and nonparallel version (Ondrej Bojar) +# 27 Jul 2006 adding the safesystem() function to handle with process failure +# 22 Jul 2006 fixed a bug about handling relative path of configuration file (Nicola Bertoldi) +# 21 Jul 2006 adapted for Moses-in-parallel (Nicola Bertoldi) +# 18 Jul 2006 adapted for Moses and cleaned up (PK) +# 21 Jan 2005 unified various versions, thorough cleanup (DWC) +# now indexing accumulated n-best list solely by feature vectors +# 14 Dec 2004 reimplemented find_threshold_points in C (NMD) +# 25 Oct 2004 Use either average or shortest (default) reference +# length as effective reference length (DWC) +# 13 Oct 2004 Use alternative decoders (DWC) +# Original version by Philipp Koehn + +use strict; +use FindBin qw($Bin); +use File::Basename; +use File::Path; +use File::Spec; +use Cwd; + +my $SCRIPTS_ROOTDIR = $Bin; +$SCRIPTS_ROOTDIR =~ s/\/training$//; +$SCRIPTS_ROOTDIR = $ENV{"SCRIPTS_ROOTDIR"} if defined($ENV{"SCRIPTS_ROOTDIR"}); + +## We preserve this bit of comments to keep the traditional weight ranges. +# "w" => [ [ 0.0, -1.0, 1.0 ] ], # word penalty +# "d" => [ [ 1.0, 0.0, 2.0 ] ], # lexicalized reordering model +# "lm" => [ [ 1.0, 0.0, 2.0 ] ], # language model +# "g" => [ [ 1.0, 0.0, 2.0 ], # generation model +# [ 1.0, 0.0, 2.0 ] ], +# "tm" => [ [ 0.3, 0.0, 0.5 ], # translation model +# [ 0.2, 0.0, 0.5 ], +# [ 0.3, 0.0, 0.5 ], +# [ 0.2, 0.0, 0.5 ], +# [ 0.0,-1.0, 1.0 ] ], # ... last weight is phrase penalty +# "lex"=> [ [ 0.1, 0.0, 0.2 ] ], # global lexical model +# "I" => [ [ 0.0,-1.0, 1.0 ] ], # input lattice scores + + + +# moses.ini file uses FULL names for lambdas, while this training script +# internally (and on the command line) uses ABBR names. +my @ABBR_FULL_MAP = qw(d=weight-d lm=weight-l tm=weight-t w=weight-w + g=weight-generation lex=weight-lex I=weight-i); +my %ABBR2FULL = map {split/=/,$_,2} @ABBR_FULL_MAP; +my %FULL2ABBR = map {my ($a, $b) = split/=/,$_,2; ($b, $a);} @ABBR_FULL_MAP; + +my $minimum_required_change_in_weights = 0.00001; + # stop if no lambda changes more than this + +my $verbose = 0; +my $usage = 0; # request for --help + +# We assume that if you don't specify working directory, +# we set the default is set to `pwd`/mert-work +#@@# my $___WORKING_DIR = File::Spec->catfile(Cwd::getcwd(), "mert-work"); +my $___WORKING_DIR = undef; +my $___DEV_F = undef; # required, input text to decode +my $___DEV_E = undef; # required, basename of files with references +my $___DECODER = undef; # required, pathname to the decoder executable +my $___CONFIG = undef; # required, pathname to startup ini file +my $___N_BEST_LIST_SIZE = 100; +my $___LATTICE_SAMPLES = 0; +my $submithost = ""; +my $queue_flags = "-hard"; # extra parameters for parallelizer + # the -l ws0ssmt was relevant only to JHU 2006 workshop +my $___JOBS = undef; # if parallel, number of jobs to use (undef or 0 -> serial) +my $___DECODER_FLAGS = ""; # additional parametrs to pass to the decoder +my $continue = 0; # should we try to continue from the last saved step? +my $skip_decoder = 0; # and should we skip the first decoder run (assuming we got interrupted during mert) +my $___FILTER_PHRASE_TABLE = 1; # filter phrase table +my $___PREDICTABLE_SEEDS = 0; +my $___START_WITH_HISTORIC_BESTS = 0; # use best settings from all previous iterations as starting points [Foster&Kuhn,2009] +my $___RANDOM_DIRECTIONS = 0; # search in random directions only +my $___NUM_RANDOM_DIRECTIONS = 0; # number of random directions, also works with default optimizer [Cer&al.,2008] +my $___PAIRWISE_RANKED_OPTIMIZER = 0; # use Hopkins&May[2011] +my $___PRO_STARTING_POINT = 0; # get a starting point from pairwise ranked optimizer +my $___RANDOM_RESTARTS = 20; +my $___HISTORIC_INTERPOLATION = 0; # interpolate optimize weights with previous iteration's weights [Hopkins&May,2011,5.4.3] +my $__THREADS = 0; +my $run = 0; + +# Parameter for effective reference length when computing BLEU score +# Default is to use shortest reference +# Use "--shortest" to use shortest reference length +# Use "--average" to use average reference length +# Use "--closest" to use closest reference length +# Only one between --shortest, --average and --closest can be set +# If more than one choice the defualt (--shortest) is used +my $___SHORTEST = 0; +my $___AVERAGE = 0; +my $___CLOSEST = 0; + +# Use "--nocase" to compute case-insensitive scores +my $___NOCASE = 0; + +# Use "--nonorm" to non normalize translation before computing scores +my $___NONORM = 0; + +# set 0 if input type is text, set 1 if input type is confusion network +my $___INPUTTYPE = 0; + + +my $mertdir = undef; # path to new mert directory +my $mertargs = undef; # args to pass through to mert & extractor +my $mertmertargs = undef; # args to pass through to mert only +my $extractorargs = undef; # args to pass through to extractor only +my $filtercmd = undef; # path to filter-model-given-input.pl +my $filterfile = undef; +my $qsubwrapper = undef; +my $qsubwrapper_exit = undef; +my $moses_parallel_cmd = undef; +my $old_sge = 0; # assume sge<6.0 +my $___CONFIG_ORIG = undef; # pathname to startup ini file before filtering +my $___ACTIVATE_FEATURES = undef; # comma-separated (or blank-separated) list of features to work on + # if undef work on all features + # (others are fixed to the starting values) +my $___RANGES = undef; +my $prev_aggregate_nbl_size = -1; # number of previous step to consider when loading data (default =-1) + # -1 means all previous, i.e. from iteration 1 + # 0 means no previous data, i.e. from actual iteration + # 1 means 1 previous data , i.e. from the actual iteration and from the previous one + # and so on +my $maximum_iterations = 25; + +##################### +my $processfeatlistcmd = undef; +my $processfeatlistargs = undef; +my $createconfigcmd = undef; +my $createconfigargs = undef; +my $decoderargs = undef; +##################### + +use Getopt::Long; +GetOptions( + "working-dir=s" => \$___WORKING_DIR, + "input=s" => \$___DEV_F, + "inputtype=i" => \$___INPUTTYPE, + "refs=s" => \$___DEV_E, + "decoder=s" => \$___DECODER, + "config=s" => \$___CONFIG, + "nbest=i" => \$___N_BEST_LIST_SIZE, + "lattice-samples=i" => \$___LATTICE_SAMPLES, + "submithost=s" => \$submithost, + "queue-flags=s" => \$queue_flags, + "jobs=i" => \$___JOBS, + "decoder-flags=s" => \$___DECODER_FLAGS, + "continue" => \$continue, + "skip-decoder" => \$skip_decoder, + "shortest" => \$___SHORTEST, + "average" => \$___AVERAGE, + "closest" => \$___CLOSEST, + "nocase" => \$___NOCASE, + "nonorm" => \$___NONORM, + "help" => \$usage, + "verbose" => \$verbose, + "mertdir=s" => \$mertdir, + "mertargs=s" => \$mertargs, + "extractorargs=s" => \$extractorargs, + "mertmertargs=s" => \$mertmertargs, + "rootdir=s" => \$SCRIPTS_ROOTDIR, + "filtercmd=s" => \$filtercmd, # allow to override the default location + "filterfile=s" => \$filterfile, # input to filtering script (useful for lattices/confnets) + "qsubwrapper=s" => \$qsubwrapper, # allow to override the default location + "mosesparallelcmd=s" => \$moses_parallel_cmd, # allow to override the default location + "old-sge" => \$old_sge, #passed to moses-parallel + "filter-phrase-table!" => \$___FILTER_PHRASE_TABLE, # (dis)allow of phrase tables + "predictable-seeds" => \$___PREDICTABLE_SEEDS, # make random restarts deterministic + "historic-bests" => \$___START_WITH_HISTORIC_BESTS, # use best settings from all previous iterations as starting points + "random-directions" => \$___RANDOM_DIRECTIONS, # search only in random directions + "run=i" => \$run, + "number-of-random-directions=i" => \$___NUM_RANDOM_DIRECTIONS, # number of random directions + "random-restarts=i" => \$___RANDOM_RESTARTS, # number of random restarts + "activate-features=s" => \$___ACTIVATE_FEATURES, #comma-separated (or blank-separated) list of features to work on (others are fixed to the starting values) + "range=s@" => \$___RANGES, + "prev-aggregate-nbestlist=i" => \$prev_aggregate_nbl_size, #number of previous step to consider when loading data (default =-1, i.e. all previous) + "maximum-iterations=i" => \$maximum_iterations, + "pairwise-ranked" => \$___PAIRWISE_RANKED_OPTIMIZER, + "pro-starting-point" => \$___PRO_STARTING_POINT, + "historic-interpolation=f" => \$___HISTORIC_INTERPOLATION, + "threads=i" => \$__THREADS +) or exit(1); + +# the 4 required parameters can be supplied on the command line directly +# or using the --options +if (scalar @ARGV == 2) { + # required parameters: input_file references_basename decoder_executable +# $___DEV_F = shift; + $___DEV_E = shift; +# $___DECODER = shift; + $___CONFIG = shift; +} + +# if ($usage || !defined $___DEV_F || !defined $___DEV_E || !defined $___DECODER || !defined $___CONFIG) { +if ($usage || !defined $___CONFIG || !defined $___DEV_E ){ + print STDERR "usage: $0 reference decoder.ini +Options: + --working-dir=mert-dir ... where all the files are created + --nbest=100 ... how big nbestlist to generate + --lattice-samples ... how many lattice samples (Chatterjee & Cancedda, emnlp 2010) + --jobs=N ... set this to anything to run moses in parallel + --mosesparallelcmd=STR ... use a different script instead of moses-parallel + --queue-flags=STRING ... anything you with to pass to qsub, eg. + '-l ws06osssmt=true'. The default is: '-hard' + To reset the parameters, please use + --queue-flags=' ' + (i.e. a space between the quotes). + --decoder-flags=STRING ... extra parameters for the decoder + --continue ... continue from the last successful iteration + --skip-decoder ... skip the decoder run for the first time, + assuming that we got interrupted during + optimization + --shortest --average --closest + ... Use shortest/average/closest reference length + as effective reference length (mutually exclusive) + --nocase ... Do not preserve case information; i.e. + case-insensitive evaluation (default is false). + --nonorm ... Do not use text normalization (flag is not active, + i.e. text is NOT normalized) + --filtercmd=STRING ... path to filter-model-given-input.pl + --filterfile=STRING ... path to alternative to input-text for filtering + model. useful for lattice decoding + --rootdir=STRING ... where do helpers reside (if not given explicitly) + --mertdir=STRING ... path to new mert implementation + --mertargs=STRING ... extra args for both extractor and mert + --extractorargs=STRING ... extra args for extractor only + --mertmertargs=STRING ... extra args for mert only + --scorenbestcmd=STRING ... path to score-nbest.py + --old-sge ... passed to parallelizers, assume Grid Engine < 6.0 + --inputtype=[0|1|2] ... Handle different input types: (0 for text, + 1 for confusion network, 2 for lattices, + default is 0) + --no-filter-phrase-table ... disallow filtering of phrase tables + (useful if binary phrase tables are available) + --random-restarts=INT ... number of random restarts (default: 20) + --predictable-seeds ... provide predictable seeds to mert so that random + restarts are the same on every run + --range=tm:0..1,-1..1 ... specify min and max value for some features + --range can be repeated as needed. + The order of the various --range specifications + is important only within a feature name. + E.g.: + --range=tm:0..1,-1..1 --range=tm:0..2 + is identical to: + --range=tm:0..1,-1..1,0..2 + but not to: + --range=tm:0..2 --range=tm:0..1,-1..1 + --activate-features=STRING ... comma-separated list of features to optimize, + others are fixed to the starting values + default: optimize all features + example: tm_0,tm_4,d_0 + --prev-aggregate-nbestlist=INT ... number of previous step to consider when + loading data (default = $prev_aggregate_nbl_size) + -1 means all previous, i.e. from iteration 1 + 0 means no previous data, i.e. only the + current iteration + N means this and N previous iterations + + --maximum-iterations=ITERS ... Maximum number of iterations. Default: $maximum_iterations + --random-directions ... search only in random directions + --number-of-random-directions=int ... number of random directions + (also works with regular optimizer, default: 0) + --pairwise-ranked ... Use PRO for optimisation (Hopkins and May, emnlp 2011) + --pro-starting-point ... Use PRO to get a starting point for MERT + --threads=NUMBER ... Use multi-threaded mert (must be compiled in). + --historic-interpolation ... Interpolate optimized weights with prior iterations' weight + (parameter sets factor [0;1] given to current weights) +"; + exit 1; +} + + +# Check validity of input parameters and set defaults if needed + +print STDERR "Using SCRIPTS_ROOTDIR: $SCRIPTS_ROOTDIR\n"; + +# path of script for filtering phrase tables and running the decoder +$filtercmd="$SCRIPTS_ROOTDIR/training/filter-model-given-input.pl" if !defined $filtercmd; + +if ( ! -x $filtercmd && ! $___FILTER_PHRASE_TABLE) { + print STDERR "Filtering command not found: $filtercmd.\n"; + print STDERR "Use --filtercmd=PATH to specify a valid one or --no-filter-phrase-table\n"; + exit 1; +} + +# $qsubwrapper="$SCRIPTS_ROOTDIR/generic/qsub-wrapper.pl" if !defined $qsubwrapper; +$qsubwrapper = "$SCRIPTS_ROOTDIR/generic/qsub-wrapper-sge-nosync.pl" if !defined $qsubwrapper; + +$qsubwrapper_exit = "$SCRIPTS_ROOTDIR/generic/qsub-wrapper-exit-sge-nosync.pl" if !defined $qsubwrapper_exit; + +# $moses_parallel_cmd = "$SCRIPTS_ROOTDIR/generic/moses-parallel.pl" +# if !defined $moses_parallel_cmd; +$moses_parallel_cmd = "$SCRIPTS_ROOTDIR/generic/moses-parallel-sge-nosync.pl" + if !defined $moses_parallel_cmd; + +if (!defined $mertdir) { + $mertdir = "$SCRIPTS_ROOTDIR/../mert"; + print STDERR "Assuming --mertdir=$mertdir\n"; +} + +my $mert_extract_cmd = "$mertdir/extractor"; +my $mert_mert_cmd = "$mertdir/mert"; +my $mert_pro_cmd = "$mertdir/pro"; + +die "Not executable: $mert_extract_cmd" if ! -x $mert_extract_cmd; +die "Not executable: $mert_mert_cmd" if ! -x $mert_mert_cmd; +die "Not executable: $mert_pro_cmd" if ! -x $mert_pro_cmd; + +my $pro_optimizer = "$mertdir/megam_i686.opt"; # or set to your installation +if (($___PAIRWISE_RANKED_OPTIMIZER || $___PRO_STARTING_POINT) && ! -x $pro_optimizer) { + print "did not find $pro_optimizer, installing it in $mertdir\n"; + `cd $mertdir; wget http://www.cs.utah.edu/~hal/megam/megam_i686.opt.gz;`; + `gunzip $pro_optimizer.gz`; + `chmod +x $pro_optimizer`; + die("ERROR: Installation of megam_i686.opt failed! Install by hand from http://www.cs.utah.edu/~hal/megam/") unless -x $pro_optimizer; +} + +$mertargs = "" if !defined $mertargs; + +my $scconfig = undef; +if ($mertargs =~ /\-\-scconfig\s+(.+?)(\s|$)/){ + $scconfig=$1; + $scconfig =~ s/\,/ /g; + $mertargs =~ s/\-\-scconfig\s+(.+?)(\s|$)//; +} + +# handling reference lengh strategy +if (($___CLOSEST + $___AVERAGE + $___SHORTEST) > 1){ + die "You can specify just ONE reference length strategy (closest or shortest or average) not both\n"; +} + +if ($___SHORTEST){ + $scconfig .= " reflen:shortest"; +}elsif ($___AVERAGE){ + $scconfig .= " reflen:average"; +}elsif ($___CLOSEST){ + $scconfig .= " reflen:closest"; +} + +# handling case-insensitive flag +if ($___NOCASE) { + $scconfig .= " case:false"; +}else{ + $scconfig .= " case:true"; +} +$scconfig =~ s/^\s+//; +$scconfig =~ s/\s+$//; +$scconfig =~ s/\s+/,/g; + +$scconfig = "--scconfig $scconfig" if ($scconfig); + +my $mert_extract_args=$mertargs; +$mert_extract_args .=" $scconfig"; +$mert_extract_args .=" $extractorargs"; + +$mertmertargs = "" if !defined $mertmertargs; + +my $mert_mert_args="$mertargs $mertmertargs"; +$mert_mert_args =~ s/\-+(binary|b)\b//; +$mert_mert_args .=" $scconfig"; +if ($___ACTIVATE_FEATURES){ $mert_mert_args .=" -o \"$___ACTIVATE_FEATURES\""; } + +my ($just_cmd_filtercmd,$x) = split(/ /,$filtercmd); +die "Not executable: $just_cmd_filtercmd" if ! -x $just_cmd_filtercmd; +die "Not executable: $moses_parallel_cmd" if defined $___JOBS && ! -x $moses_parallel_cmd; +die "Not executable: $qsubwrapper" if defined $___JOBS && ! -x $qsubwrapper; +#@@# die "Not executable: $___DECODER" if ! -x $___DECODER; + +#@@# my $input_abs = ensure_full_path($___DEV_F); +#@@# die "File not found: $___DEV_F (interpreted as $input_abs)." +#@@# if ! -e $input_abs; +#@@# $___DEV_F = $input_abs; + +# Option to pass to qsubwrapper and moses-parallel +my $pass_old_sge = $old_sge ? "-old-sge" : ""; + +#@@# my $decoder_abs = ensure_full_path($___DECODER); +#@@# die "File not executable: $___DECODER (interpreted as $decoder_abs)." +#@@# if ! -x $decoder_abs; +#@@# $___DECODER = $decoder_abs; + +my $ref_abs = ensure_full_path($___DEV_E); +# check if English dev set (reference translations) exist and store a list of all references +my @references; +if (-e $ref_abs) { + push @references, $ref_abs; +} +else { + # if multiple file, get a full list of the files + my $part = 0; + if (! -e $ref_abs."0" && -e $ref_abs.".ref0") { + $ref_abs .= ".ref"; + } + while (-e $ref_abs.$part) { + push @references, $ref_abs.$part; + $part++; + } + die("Reference translations not found: $___DEV_E (interpreted as $ref_abs)") unless $part; +} + +my $config_abs = ensure_full_path($___CONFIG); +die "File not found: $___CONFIG (interpreted as $config_abs)." + if ! -e $config_abs; +$___CONFIG = $config_abs; + +# moses should use our config +if ($___DECODER_FLAGS =~ /(^|\s)-(config|f) / +|| $___DECODER_FLAGS =~ /(^|\s)-(ttable-file|t) / +|| $___DECODER_FLAGS =~ /(^|\s)-(distortion-file) / +|| $___DECODER_FLAGS =~ /(^|\s)-(generation-file) / +|| $___DECODER_FLAGS =~ /(^|\s)-(lmodel-file) / +|| $___DECODER_FLAGS =~ /(^|\s)-(global-lexical-file) / +) { + die "It is forbidden to supply any of -config, -ttable-file, -distortion-file, -generation-file or -lmodel-file in the --decoder-flags.\nPlease use only the --config option to give the config file that lists all the supplementary files."; +} + +# as weights are normalized in the next steps (by cmert) +# normalize initial LAMBDAs, too +my $need_to_normalize = 1; + +#store current directory and create the working directory (if needed) +my $cwd = `pawd 2>/dev/null`; +if(!$cwd){$cwd = `pwd`;} +chomp($cwd); + +$___WORKING_DIR = $cwd if (!defined $___WORKING_DIR); +chomp $___WORKING_DIR; + +print STDERR "working dir is $___WORKING_DIR\n"; +#@@# mkpath($___WORKING_DIR); + +{ +# open local scope + +#chdir to the working directory +chdir($___WORKING_DIR) or die "Can't chdir to $___WORKING_DIR"; + +# fixed file names +my $mert_outfile = "mert.out"; +my $mert_logfile = "mert.log"; +my $weights_in_file = "init.opt"; +my $weights_out_file = "weights.txt"; + +# set start run +my $start_run = 1; +my $bestpoint = undef; +my $devbleu = undef; +my $sparse_weights_file = undef; +my $jobid = -1; + +my $prev_feature_file = undef; +my $prev_score_file = undef; +my $prev_init_file = undef; + + +######################### +# set jobid to trace different jobs +my $prevjid = undef; + + + +#### my $run=$start_run-1; + +my $oldallsorted = undef; +my $allsorted = undef; + +my $nbest_file=undef; +my $lsamp_file=undef; #Lattice samples +my $orig_nbest_file=undef; # replaced if lattice sampling +my $cmd=undef; + + + + + + $nbest_file="run$run.best$___N_BEST_LIST_SIZE.out"; + safesystem("gzip -f $nbest_file") or die "Failed to gzip run*out"; + $nbest_file = $nbest_file.".gz"; + + + # extract score statistics and features from the nbest lists + print STDERR "Scoring the nbestlist.\n"; + + my $base_feature_file = "features.dat"; + my $base_score_file = "scores.dat"; + my $feature_file = "run$run.${base_feature_file}"; + my $score_file = "run$run.${base_score_file}"; + + $cmd = "$mert_extract_cmd $mert_extract_args --scfile $score_file --ffile $feature_file -r ".join(",", @references)." -n $nbest_file"; + $cmd = create_extractor_script($cmd, $___WORKING_DIR); + + &submit_or_exec($cmd,"extract.out","extract.err","extract.id"); + + + +} # end of local scope + +sub get_weights_from_mert { + my ($outfile,$logfile,$weight_count,$sparse_weights) = @_; + my ($bestpoint,$devbleu); + if ($___PAIRWISE_RANKED_OPTIMIZER || ($___PRO_STARTING_POINT && $logfile =~ /pro/)) { + open(IN,$outfile) or die "Can't open $outfile"; + my (@WEIGHT,$sum); + for(my $i=0;$i<$weight_count;$i++) { push @WEIGHT, 0; } + while() { + # regular features + if (/^F(\d+) ([\-\.\de]+)/) { + $WEIGHT[$1] = $2; + $sum += abs($2); + } + # sparse features + elsif(/^(.+_.+) ([\-\.\de]+)/) { + $$sparse_weights{$1} = $2; + } + } + $devbleu = "unknown"; + foreach (@WEIGHT) { $_ /= $sum; } + foreach (keys %{$sparse_weights}) { $$sparse_weights{$_} /= $sum; } + $bestpoint = join(" ",@WEIGHT); + close IN; + } + else { + open(IN,$logfile) or die "Can't open $logfile"; + while () { + if (/Best point:\s*([\s\d\.\-e]+?)\s*=> ([\-\d\.]+)/) { + $bestpoint = $1; + $devbleu = $2; + last; + } + } + close IN; + } + return ($bestpoint,$devbleu); +} + + + +sub sanity_check_order_of_lambdas { + my $featlist = shift; + my $filename_or_stream = shift; + + my @expected_lambdas = @{$featlist->{"names"}}; + my @got = get_order_of_scores_from_nbestlist($filename_or_stream); + die "Mismatched lambdas. Decoder returned @got, we expected @expected_lambdas" + if "@got" ne "@expected_lambdas"; +} + +sub get_featlist_from_moses { + # run moses with the given config file and return the list of features and + # their initial values + my $configfn = shift; + my $featlistfn = "./features.list"; + if (-e $featlistfn) { + print STDERR "Using cached features list: $featlistfn\n"; + } else { + print STDERR "Asking moses for feature names and values from $___CONFIG\n"; + my $cmd = "$___DECODER $___DECODER_FLAGS -config $configfn -inputtype $___INPUTTYPE -show-weights > $featlistfn"; + safesystem($cmd) or die "Failed to run moses with the config $configfn"; + } + + # read feature list + my @names = (); + my @startvalues = (); + open(INI,$featlistfn) or die "Can't read $featlistfn"; + my $nr = 0; + my @errs = (); + while () { + $nr++; + chomp; + /^(.+) (\S+) (\S+)$/ || die("invalid feature: $_"); + my ($longname, $feature, $value) = ($1,$2,$3); + next if $value eq "sparse"; + push @errs, "$featlistfn:$nr:Bad initial value of $feature: $value\n" + if $value !~ /^[+-]?[0-9.e]+$/; + push @errs, "$featlistfn:$nr:Unknown feature '$feature', please add it to \@ABBR_FULL_MAP\n" + if !defined $ABBR2FULL{$feature}; + push @names, $feature; + push @startvalues, $value; + } + close INI; + if (scalar @errs) { + print STDERR join("", @errs); + exit 1; + } + return {"names"=>\@names, "values"=>\@startvalues}; +} + +sub get_order_of_scores_from_nbestlist { + # read the first line and interpret the ||| label: num num num label2: num ||| column in nbestlist + # return the score labels in order + my $fname_or_source = shift; + # print STDERR "Peeking at the beginning of nbestlist to get order of scores: $fname_or_source\n"; + open IN, $fname_or_source or die "Failed to get order of scores from nbestlist '$fname_or_source'"; + my $line = ; + close IN; + die "Line empty in nbestlist '$fname_or_source'" if !defined $line; + my ($sent, $hypo, $scores, $total) = split /\|\|\|/, $line; + $scores =~ s/^\s*|\s*$//g; + die "No scores in line: $line" if $scores eq ""; + + my @order = (); + my $label = undef; + my $sparse = 0; # we ignore sparse features here + foreach my $tok (split /\s+/, $scores) { + if ($tok =~ /.+_.+:/) { + $sparse = 1; + } elsif ($tok =~ /^([a-z][0-9a-z]*):/i) { + $label = $1; + } elsif ($tok =~ /^-?[-0-9.e]+$/) { + if (!$sparse) { + # a score found, remember it + die "Found a score but no label before it! Bad nbestlist '$fname_or_source'!" + if !defined $label; + push @order, $label; + } + $sparse = 0; + } else { + die "Not a label, not a score '$tok'. Failed to parse the scores string: '$scores' of nbestlist '$fname_or_source'"; + } + } + print STDERR "The decoder returns the scores in this order: @order\n"; + return @order; +} + + +sub safesystem { + print STDERR "Executing: @_\n"; + system(@_); + if ($? == -1) { + print STDERR "Failed to execute: @_\n $!\n"; + exit(1); + } + elsif ($? & 127) { + printf STDERR "Execution of: @_\n died with signal %d, %s coredump\n", + ($? & 127), ($? & 128) ? 'with' : 'without'; + exit(1); + } + else { + my $exitcode = $? >> 8; + print STDERR "Exit code: $exitcode\n" if $exitcode; + return ! $exitcode; + } +} +sub ensure_full_path { + my $PATH = shift; +$PATH =~ s/\/nfsmnt//; + return $PATH if $PATH =~ /^\//; + my $dir = `pawd 2>/dev/null`; + if(!$dir){$dir = `pwd`;} + chomp($dir); + $PATH = $dir."/".$PATH; + $PATH =~ s/[\r\n]//g; + $PATH =~ s/\/\.\//\//g; + $PATH =~ s/\/+/\//g; + my $sanity = 0; + while($PATH =~ /\/\.\.\// && $sanity++<10) { + $PATH =~ s/\/+/\//g; + $PATH =~ s/\/[^\/]+\/\.\.\//\//g; + } + $PATH =~ s/\/[^\/]+\/\.\.$//; + $PATH =~ s/\/+$//; +$PATH =~ s/\/nfsmnt//; + return $PATH; +} + +sub submit_or_exec { + + my $argvlen = @_; + my $cmd = undef; + my $stdout = undef; + my $stderr = undef; + my $jidfile = undef; + my $prevjid = undef; + + # if supply 3 arguments, exec without submit + # if supply 4 arguments, then submit new job + # if supply 5 arguments, wait for the previous job to finish + if ($argvlen == 3){ + ($cmd,$stdout,$stderr) = @_; + } elsif ($argvlen == 4){ + ($cmd,$stdout,$stderr,$jidfile) = @_; + } elsif ($argvlen == 5){ + ($cmd,$stdout,$stderr,$jidfile,$prevjid) = @_; + } + + print STDERR "exec: $cmd\n"; + if (defined $___JOBS && $___JOBS > 0 && $argvlen==5) { + safesystem("$qsubwrapper $pass_old_sge -command='$cmd' -queue-parameters=\"$queue_flags\" -stdout=$stdout -stderr=$stderr -jidfile=$jidfile -prevjid=$prevjid" ) + or die "ERROR: Failed to submit '$cmd' (via $qsubwrapper)"; + } + elsif (defined $___JOBS && $___JOBS > 0 && $argvlen==4) { + safesystem("$qsubwrapper $pass_old_sge -command='$cmd' -queue-parameters=\"$queue_flags\" -stdout=$stdout -stderr=$stderr -jidfile=$jidfile" ) + or die "ERROR: Failed to submit '$cmd' (via $qsubwrapper)"; + } else { + safesystem("$cmd > $stdout 2> $stderr") or die "ERROR: Failed to run '$cmd'."; + } +} + +sub exit_submit { + + my $argvlen = @_; + my $cmd = undef; + my $stdout = undef; + my $stderr = undef; + my $jidfile = undef; + my $pidfile = undef; + my $prevjid = undef; + my $prevjidarraysize = 0; + my @prevjidarray = (); + my $pid = undef; + my $qsubcmd=""; + my $hj=""; + + # if supply 4 arguments, then submit new job + # if supply 5 arguments, wait for the previous job to finish + if ($argvlen == 2) { + ($stdout,$stderr) = @_; + } elsif ($argvlen == 4){ + ($stdout,$stderr,$jidfile,$pidfile) = @_; + } elsif ($argvlen == 5){ + ($stdout,$stderr,$jidfile,$pidfile,$prevjid) = @_; + } + + # parse prevjid ######################## + $prevjid =~ s/^\s+|\s+$//g; + @prevjidarray = split(/\s+/,$prevjid); + $prevjidarraysize = scalar(@prevjidarray); + ######################################## + + + # print STDERR "exec: $stdout\n"; + + # read pid from file, and draft exit script ################## + chomp ($pid=`tail -n 1 $pidfile`); + open (OUT, ">exitjob$pid.sh"); + + my $scriptheader="\#\!/bin/bash\n\#\$ -S /bin/sh\n# Both lines are needed to invoke base\n#the above line is ignored by qsub, unless parameter \"-b yes\" is set!\n\n"; + $scriptheader .="uname -a\n\n"; + $scriptheader .="cd $___WORKING_DIR\n\n"; + + print OUT $scriptheader; + + print OUT "if $qsubwrapper_exit -submithost=$submithost -stdout=$stdout -stderr=$stderr -jidfile=$jidfile -pidfile=$pidfile > exitjob$pid.out 2> exitjob$pid.err ; then + echo 'succeeded' +else + echo failed with exit status \$\? + die=1 +fi +"; + print OUT "\n\n"; + + close (OUT); + # setting permissions of the script + chmod(oct(755),"exitjob$pid.sh"); + ############################################################## + + + if (defined $___JOBS && $___JOBS > 0 && $argvlen==5) { + if (defined $prevjid && $prevjid!=-1 && $prevjidarraysize == 1){ + $hj = "-hold_jid $prevjid"; + } elsif (defined $prevjid && $prevjidarraysize > 1){ + $hj = "-hold_jid " . join(" -hold_jid ", @prevjidarray); + } + $qsubcmd="qsub $queue_flags -V $hj exitjob$pid.sh > exitjob$pid.log 2>&1"; + safesystem($qsubcmd) or die "ERROR: Failed to exit-submit $pid (via $qsubwrapper_exit)"; + } elsif (defined $___JOBS && $___JOBS > 0 && $argvlen==4) { + $qsubcmd="qsub $queue_flags -V exitjob$pid.sh > exitjob$pid.log 2>&1"; + safesystem($qsubcmd) or die "ERROR: Failed to exit-submit $pid (via $qsubwrapper_exit)"; + } else { + safesystem("rm $stdout") or die "ERROR: Failed to remove '$stdout'."; + safesystem("rm $stderr") or die "ERROR: Failed to remove '$stderr'."; + } +} + + + + + + + +sub create_extractor_script() +{ + my ($cmd, $outdir) = @_; + my $script_path = File::Spec->catfile($outdir, "extractor.sh"); + + open my $out, '>', $script_path + or die "Couldn't open $script_path for writing: $!\n"; + print $out "#!/bin/bash\n"; + print $out "cd $outdir\n"; + print $out "$cmd\n"; + close($out); + + `chmod +x $script_path`; + + return $script_path; +} diff --git a/mosesdecoder/contrib/other-builds/CreateProbingPT/.cproject b/mosesdecoder/contrib/other-builds/CreateProbingPT/.cproject new file mode 100644 index 0000000000000000000000000000000000000000..a06a01f4a68d00bcf31c69be0e1c6867f65614ac --- /dev/null +++ b/mosesdecoder/contrib/other-builds/CreateProbingPT/.cproject @@ -0,0 +1,150 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mosesdecoder/contrib/other-builds/CreateProbingPT/.project b/mosesdecoder/contrib/other-builds/CreateProbingPT/.project new file mode 100644 index 0000000000000000000000000000000000000000..6e863a77ef8526c7a2476c1555b74c2b3346f0b6 --- /dev/null +++ b/mosesdecoder/contrib/other-builds/CreateProbingPT/.project @@ -0,0 +1,36 @@ + + + CreateProbingPT + + + moses + util + + + + org.eclipse.cdt.managedbuilder.core.genmakebuilder + clean,full,incremental, + + + + + org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder + full,incremental, + + + + + + org.eclipse.cdt.core.cnature + org.eclipse.cdt.core.ccnature + org.eclipse.cdt.managedbuilder.core.managedBuildNature + org.eclipse.cdt.managedbuilder.core.ScannerConfigNature + + + + CreateProbingPT.cpp + 1 + PARENT-3-PROJECT_LOC/probingpt/CreateProbingPT.cpp + + + diff --git a/mosesdecoder/contrib/other-builds/MosesManagedDLL/ManagedMoses.cpp b/mosesdecoder/contrib/other-builds/MosesManagedDLL/ManagedMoses.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f897625c49372bb90b9afe287b9cf0f0149a1b39 --- /dev/null +++ b/mosesdecoder/contrib/other-builds/MosesManagedDLL/ManagedMoses.cpp @@ -0,0 +1,28 @@ +#include +#include "Moses2Wrapper.h" + +using namespace System; +using namespace msclr::interop; + +//TODO: include headers as per the build process +namespace Moses { + public ref class Moses2Wrapper + { + public: + Moses2Wrapper(String^ filePath) { + const std::string standardString = marshal_as(filePath); + m_pWrapper = new Moses2::Moses2Wrapper(standardString); + } + ~Moses2Wrapper() { this->!Moses2Wrapper(); } + String^ Translate(String^ input, long requestId) { + const std::string standardString = marshal_as(input); + std::string output = m_pWrapper->Translate(standardString, requestId); + String^ str = gcnew String(output.c_str()); + return str; + } + protected: + !Moses2Wrapper() { delete m_pWrapper; m_pWrapper = nullptr; } + private: + Moses2::Moses2Wrapper *m_pWrapper; + }; +} \ No newline at end of file diff --git a/mosesdecoder/contrib/other-builds/MosesManagedDLL/MosesManagedDLL.vcxproj b/mosesdecoder/contrib/other-builds/MosesManagedDLL/MosesManagedDLL.vcxproj new file mode 100644 index 0000000000000000000000000000000000000000..804bcc011bcd497ba61a2956232077d13314fee1 --- /dev/null +++ b/mosesdecoder/contrib/other-builds/MosesManagedDLL/MosesManagedDLL.vcxproj @@ -0,0 +1,170 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + 16.0 + Win32Proj + {5fb67144-27c9-4993-a084-e2097ffcc4fe} + MosesManagedDLL + 10.0 + + + + DynamicLibrary + true + v142 + Unicode + + + DynamicLibrary + false + v142 + true + Unicode + + + DynamicLibrary + true + v142 + Unicode + true + + + DynamicLibrary + false + v142 + true + Unicode + + + + + + + + + + + + + + + + + + + + + true + + + false + + + true + + + false + + + + Level3 + true + WIN32;_DEBUG;MOSESMANAGEDDLL_EXPORTS;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) + true + Use + pch.h + + + Windows + true + false + + + + + Level3 + true + true + true + WIN32;NDEBUG;MOSESMANAGEDDLL_EXPORTS;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) + true + Use + pch.h + + + Windows + true + true + true + false + + + + + Level3 + true + _DEBUG;MOSESMANAGEDDLL_EXPORTS;NO_COMPACT_TABLES;HAVE_CMPHXXX;MAX_NUM_FACTORS=4;KENLM_MAX_ORDER=6;_USE_MATH_DEFINES;NOMINMAX;WITH_THREADS;NO_PIPES;_CONSOLE;_LIB;%(PreprocessorDefinitions) + false + NotUsing + pch.h + F:\boost_1_72_0;D:\moses-mstranslator;D:\xmlrpc-c-1.51.06\include;D:\moses-mstranslator\moses2 + true + MultiThreadedDebugDLL + + + Windows + true + false + D:\zlib-1.2.8\contrib\vstudio\vc11\x64\ZlibDllRelease;D:\xmlrpc-c-1.51.06\bin\Debug-Static-x64;C:\Users\sija\Downloads\2.0.2\bonitao-cmph-e5f83da\Debug;D:\zlib_1_2_8_msvc2015_64\msvc2015_64\lib\zlib;D:\mman-win32-master\x64\Release;F:\boost_1_72_0\lib64-msvc-14.2;%(AdditionalLibraryDirectories) + libxmlrpc.lib;libxmlrpc_server_abyss.lib;libxmlrpc_server.lib;libxmlrpc_abyss.lib;libxmlrpc_util.lib;libxmlrpc_xmlparse.lib;libxmlrpc_xmltok.lib;libxmlrpc++.lib;zlibwapi.lib;mman.lib;cmph.lib;%(AdditionalDependencies) + + + + + Level3 + true + true + true + NDEBUG;MOSESMANAGEDDLL_EXPORTS;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) + true + Use + pch.h + + + Windows + true + true + true + false + + + + + + + + {b4304e97-d37f-4022-bd03-841a4faee398} + + + + + + \ No newline at end of file diff --git a/mosesdecoder/contrib/other-builds/MosesManagedDLL/MosesManagedDLL.vcxproj.filters b/mosesdecoder/contrib/other-builds/MosesManagedDLL/MosesManagedDLL.vcxproj.filters new file mode 100644 index 0000000000000000000000000000000000000000..8bd68daf0595a8c6e3deadf0299c2bb340b1c6ad --- /dev/null +++ b/mosesdecoder/contrib/other-builds/MosesManagedDLL/MosesManagedDLL.vcxproj.filters @@ -0,0 +1,22 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;c++;cppm;ixx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;h++;hm;inl;inc;ipp;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + \ No newline at end of file diff --git a/mosesdecoder/contrib/other-builds/MosesWrapperTest/MosesWrapperTest.csproj b/mosesdecoder/contrib/other-builds/MosesWrapperTest/MosesWrapperTest.csproj new file mode 100644 index 0000000000000000000000000000000000000000..333fca400fe28fd1ba9541f516e0420b608fbfb9 --- /dev/null +++ b/mosesdecoder/contrib/other-builds/MosesWrapperTest/MosesWrapperTest.csproj @@ -0,0 +1,86 @@ + + + + + Debug + AnyCPU + {A2B603DB-F52E-4A83-BB61-F23D212E6482} + Exe + MosesWrapperTest + MosesWrapperTest + v4.7.2 + 512 + true + true + publish\ + true + Disk + false + Foreground + 7 + Days + false + false + true + 0 + 1.0.0.%2a + false + false + true + + + x64 + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 2 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + + + + {5fb67144-27c9-4993-a084-e2097ffcc4fe} + MosesManagedDLL + + + + + False + Microsoft .NET Framework 4.7.2 %28x86 and x64%29 + true + + + False + .NET Framework 3.5 SP1 + false + + + + \ No newline at end of file diff --git a/mosesdecoder/contrib/other-builds/MosesWrapperTest/Program.cs b/mosesdecoder/contrib/other-builds/MosesWrapperTest/Program.cs new file mode 100644 index 0000000000000000000000000000000000000000..97fed286382df3dacabfd7f772598e78df50c4c0 --- /dev/null +++ b/mosesdecoder/contrib/other-builds/MosesWrapperTest/Program.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Moses; +namespace MosesWrapperTest +{ + class Program { + + static void Main(string[] args) + { + Moses2Wrapper e = new Moses2Wrapper("D:/moses-mstranslator/test_sentence_with_candidates/moses_mspt.ini"); + string mystring = e.Translate("फ ो ट ो ं @@@ ट ||| a ||| 0.5338410658500136 $$$ ट ||| c ||| 0.10587171128910133 $$$ ट ||| m ||| 0.7056508746775306 $$$ ं ||| l ||| 0.29237797398236876 $$$ ं ||| o ||| 0.4026301817948226 $$$ ं ||| r ||| 0.20594041196734436 $$$ फ ||| c ||| 0.46792456587433573 $$$ फ ||| g ||| 0.43855815762641204 $$$ फ ||| x ||| 0.7077570324853759 $$$ ो ||| h ||| 0.9869239425073358 $$$ ो ||| i ||| 0.6660016809625412 $$$ ो ||| h ||| 0.8425506301302961", 123456789); + Console.WriteLine(mystring); + return; + } + } +} diff --git a/mosesdecoder/contrib/other-builds/OnDiskPt/.cproject b/mosesdecoder/contrib/other-builds/OnDiskPt/.cproject new file mode 100644 index 0000000000000000000000000000000000000000..544bb24f5e136c6ee679c646cd5ee31fe335d096 --- /dev/null +++ b/mosesdecoder/contrib/other-builds/OnDiskPt/.cproject @@ -0,0 +1,140 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mosesdecoder/contrib/other-builds/OnDiskPt/.project b/mosesdecoder/contrib/other-builds/OnDiskPt/.project new file mode 100644 index 0000000000000000000000000000000000000000..eb9f683561448735bc757e54070b78610dd7e7cd --- /dev/null +++ b/mosesdecoder/contrib/other-builds/OnDiskPt/.project @@ -0,0 +1,185 @@ + + + OnDiskPt + + + + + + org.eclipse.cdt.managedbuilder.core.genmakebuilder + clean,full,incremental, + + + ?name? + + + + org.eclipse.cdt.make.core.append_environment + true + + + org.eclipse.cdt.make.core.autoBuildTarget + all + + + org.eclipse.cdt.make.core.buildArguments + -j3 + + + org.eclipse.cdt.make.core.buildCommand + make + + + org.eclipse.cdt.make.core.buildLocation + ${workspace_loc:/OnDiskPt/Debug} + + + org.eclipse.cdt.make.core.cleanBuildTarget + clean + + + org.eclipse.cdt.make.core.contents + org.eclipse.cdt.make.core.activeConfigSettings + + + org.eclipse.cdt.make.core.enableAutoBuild + false + + + org.eclipse.cdt.make.core.enableCleanBuild + true + + + org.eclipse.cdt.make.core.enableFullBuild + true + + + org.eclipse.cdt.make.core.fullBuildTarget + all + + + org.eclipse.cdt.make.core.stopOnError + true + + + org.eclipse.cdt.make.core.useDefaultBuildCmd + true + + + + + org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder + full,incremental, + + + + + + org.eclipse.cdt.core.cnature + org.eclipse.cdt.core.ccnature + org.eclipse.cdt.managedbuilder.core.managedBuildNature + org.eclipse.cdt.managedbuilder.core.ScannerConfigNature + + + + Jamfile + 1 + PARENT-3-PROJECT_LOC/OnDiskPt/Jamfile + + + Main.cpp + 1 + PARENT-3-PROJECT_LOC/OnDiskPt/Main.cpp + + + Main.h + 1 + PARENT-3-PROJECT_LOC/OnDiskPt/Main.h + + + OnDiskWrapper.cpp + 1 + PARENT-3-PROJECT_LOC/OnDiskPt/OnDiskWrapper.cpp + + + OnDiskWrapper.h + 1 + PARENT-3-PROJECT_LOC/OnDiskPt/OnDiskWrapper.h + + + Phrase.cpp + 1 + PARENT-3-PROJECT_LOC/OnDiskPt/Phrase.cpp + + + Phrase.h + 1 + PARENT-3-PROJECT_LOC/OnDiskPt/Phrase.h + + + PhraseNode.cpp + 1 + PARENT-3-PROJECT_LOC/OnDiskPt/PhraseNode.cpp + + + PhraseNode.h + 1 + PARENT-3-PROJECT_LOC/OnDiskPt/PhraseNode.h + + + SourcePhrase.cpp + 1 + PARENT-3-PROJECT_LOC/OnDiskPt/SourcePhrase.cpp + + + SourcePhrase.h + 1 + PARENT-3-PROJECT_LOC/OnDiskPt/SourcePhrase.h + + + TargetPhrase.cpp + 1 + PARENT-3-PROJECT_LOC/OnDiskPt/TargetPhrase.cpp + + + TargetPhrase.h + 1 + PARENT-3-PROJECT_LOC/OnDiskPt/TargetPhrase.h + + + TargetPhraseCollection.cpp + 1 + PARENT-3-PROJECT_LOC/OnDiskPt/TargetPhraseCollection.cpp + + + TargetPhraseCollection.h + 1 + PARENT-3-PROJECT_LOC/OnDiskPt/TargetPhraseCollection.h + + + Vocab.cpp + 1 + PARENT-3-PROJECT_LOC/OnDiskPt/Vocab.cpp + + + Vocab.h + 1 + PARENT-3-PROJECT_LOC/OnDiskPt/Vocab.h + + + Word.cpp + 1 + PARENT-3-PROJECT_LOC/OnDiskPt/Word.cpp + + + Word.h + 1 + PARENT-3-PROJECT_LOC/OnDiskPt/Word.h + + + queryOnDiskPt.cpp + 1 + PARENT-3-PROJECT_LOC/OnDiskPt/queryOnDiskPt.cpp + + + diff --git a/mosesdecoder/contrib/other-builds/OnDiskPt/OnDiskPt.project b/mosesdecoder/contrib/other-builds/OnDiskPt/OnDiskPt.project new file mode 100644 index 0000000000000000000000000000000000000000..3a89ec8328840b58f6c9126a6cbf5f82c01c0a9f --- /dev/null +++ b/mosesdecoder/contrib/other-builds/OnDiskPt/OnDiskPt.project @@ -0,0 +1,140 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mosesdecoder/contrib/other-builds/all.workspace b/mosesdecoder/contrib/other-builds/all.workspace new file mode 100644 index 0000000000000000000000000000000000000000..5a7eaf114e8a1e748027afdd3d18d9f256a35868 --- /dev/null +++ b/mosesdecoder/contrib/other-builds/all.workspace @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mosesdecoder/contrib/other-builds/cmake/boost.example/CMakeLists.txt b/mosesdecoder/contrib/other-builds/cmake/boost.example/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..abc8b37e2896b9bc5410ae86cf533e4d57d9f4d8 --- /dev/null +++ b/mosesdecoder/contrib/other-builds/cmake/boost.example/CMakeLists.txt @@ -0,0 +1,7 @@ +cmake_minimum_required(VERSION 2.8) + +find_package(Boost 1.36.0 REQUIRED COMPONENTS locale) + +include_directories(${Boost_INCLUDE_DIRS}) +add_executable(demo main.cpp) +target_link_libraries(demo ${Boost_LIBRARIES}) diff --git a/mosesdecoder/contrib/other-builds/consolidate/.cproject b/mosesdecoder/contrib/other-builds/consolidate/.cproject new file mode 100644 index 0000000000000000000000000000000000000000..d4b35e5005c37d2065609ad642b000d275e6763a --- /dev/null +++ b/mosesdecoder/contrib/other-builds/consolidate/.cproject @@ -0,0 +1,165 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mosesdecoder/contrib/other-builds/consolidate/.project b/mosesdecoder/contrib/other-builds/consolidate/.project new file mode 100644 index 0000000000000000000000000000000000000000..db9a1fa8f298671c0eb107288b84662da7a87ffb --- /dev/null +++ b/mosesdecoder/contrib/other-builds/consolidate/.project @@ -0,0 +1,79 @@ + + + consolidate + + + lm + moses + OnDiskPt + search + util + + + + org.eclipse.cdt.managedbuilder.core.genmakebuilder + clean,full,incremental, + + + + + org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder + full,incremental, + + + + + + org.eclipse.cdt.core.cnature + org.eclipse.cdt.core.ccnature + org.eclipse.cdt.managedbuilder.core.managedBuildNature + org.eclipse.cdt.managedbuilder.core.ScannerConfigNature + + + + InputFileStream.cpp + 1 + PARENT-3-PROJECT_LOC/phrase-extract/InputFileStream.cpp + + + InputFileStream.h + 1 + PARENT-3-PROJECT_LOC/phrase-extract/InputFileStream.h + + + OutputFileStream.cpp + 1 + PARENT-3-PROJECT_LOC/phrase-extract/OutputFileStream.cpp + + + OutputFileStream.h + 1 + PARENT-3-PROJECT_LOC/phrase-extract/OutputFileStream.h + + + PropertiesConsolidator.cpp + 1 + PARENT-3-PROJECT_LOC/phrase-extract/PropertiesConsolidator.cpp + + + PropertiesConsolidator.h + 1 + PARENT-3-PROJECT_LOC/phrase-extract/PropertiesConsolidator.h + + + consolidate-main.cpp + 1 + PARENT-3-PROJECT_LOC/phrase-extract/consolidate-main.cpp + + + tables-core.cpp + 1 + PARENT-3-PROJECT_LOC/phrase-extract/tables-core.cpp + + + tables-core.h + 1 + PARENT-3-PROJECT_LOC/phrase-extract/tables-core.h + + + diff --git a/mosesdecoder/contrib/other-builds/consolidate/consolidate.project b/mosesdecoder/contrib/other-builds/consolidate/consolidate.project new file mode 100644 index 0000000000000000000000000000000000000000..25539fad4265d0ee565ebf9dddab18fca1edf3fd --- /dev/null +++ b/mosesdecoder/contrib/other-builds/consolidate/consolidate.project @@ -0,0 +1,137 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + None + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + None + + + + + + + + + + + + + + + + + + + + + + diff --git a/mosesdecoder/contrib/other-builds/extractor/.cproject b/mosesdecoder/contrib/other-builds/extractor/.cproject new file mode 100644 index 0000000000000000000000000000000000000000..728ed4410b7ceeb1c51cb3434fa2e5e7f29814e7 --- /dev/null +++ b/mosesdecoder/contrib/other-builds/extractor/.cproject @@ -0,0 +1,144 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mosesdecoder/contrib/other-builds/extractor/.project b/mosesdecoder/contrib/other-builds/extractor/.project new file mode 100644 index 0000000000000000000000000000000000000000..a720f94b4a1e639530374f33e82e31afec1799f5 --- /dev/null +++ b/mosesdecoder/contrib/other-builds/extractor/.project @@ -0,0 +1,607 @@ + + + extractor + + + mert_lib + util + + + + org.eclipse.cdt.managedbuilder.core.genmakebuilder + clean,full,incremental, + + + ?name? + + + + org.eclipse.cdt.make.core.append_environment + true + + + org.eclipse.cdt.make.core.autoBuildTarget + all + + + org.eclipse.cdt.make.core.buildArguments + + + + org.eclipse.cdt.make.core.buildCommand + make + + + org.eclipse.cdt.make.core.buildLocation + ${workspace_loc:/extractor/Debug} + + + org.eclipse.cdt.make.core.cleanBuildTarget + clean + + + org.eclipse.cdt.make.core.contents + org.eclipse.cdt.make.core.activeConfigSettings + + + org.eclipse.cdt.make.core.enableAutoBuild + false + + + org.eclipse.cdt.make.core.enableCleanBuild + true + + + org.eclipse.cdt.make.core.enableFullBuild + true + + + org.eclipse.cdt.make.core.fullBuildTarget + all + + + org.eclipse.cdt.make.core.stopOnError + true + + + org.eclipse.cdt.make.core.useDefaultBuildCmd + true + + + + + org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder + full,incremental, + + + + + + org.eclipse.cdt.core.cnature + org.eclipse.cdt.core.ccnature + org.eclipse.cdt.managedbuilder.core.managedBuildNature + org.eclipse.cdt.managedbuilder.core.ScannerConfigNature + + + + InternalTree.cpp + 1 + PARENT-3-PROJECT_LOC/mert/InternalTree.cpp + + + InternalTree.h + 1 + PARENT-3-PROJECT_LOC/mert/InternalTree.h + + + bin + 2 + virtual:/virtual + + + extractor.cpp + 1 + PARENT-3-PROJECT_LOC/mert/extractor.cpp + + + bin/gcc-4.8 + 2 + virtual:/virtual + + + bin/gcc-4.8/release + 2 + virtual:/virtual + + + bin/gcc-4.8/release/debug-symbols-on + 2 + virtual:/virtual + + + bin/gcc-4.8/release/debug-symbols-on/link-static + 2 + virtual:/virtual + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi + 2 + virtual:/virtual + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/BleuDocScorer.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/BleuDocScorer.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/BleuScorer.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/BleuScorer.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/BleuScorerTest.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/BleuScorerTest.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/CderScorer.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/CderScorer.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/Data.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/Data.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/DataTest.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/DataTest.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/FeatureArray.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/FeatureArray.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/FeatureData.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/FeatureData.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/FeatureDataIterator.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/FeatureDataIterator.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/FeatureDataTest.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/FeatureDataTest.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/FeatureStats.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/FeatureStats.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/FileStream.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/FileStream.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/GzFileBuf.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/GzFileBuf.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/HypPackEnumerator.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/HypPackEnumerator.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/InterpolatedScorer.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/InterpolatedScorer.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/MeteorScorer.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/MeteorScorer.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/MiraFeatureVector.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/MiraFeatureVector.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/MiraWeightVector.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/MiraWeightVector.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/NgramTest.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/NgramTest.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/Optimizer.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/Optimizer.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/OptimizerFactory.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/OptimizerFactory.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/OptimizerFactoryTest.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/OptimizerFactoryTest.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/PerScorer.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/PerScorer.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/Permutation.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/Permutation.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/PermutationScorer.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/PermutationScorer.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/Point.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/Point.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/PointTest.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/PointTest.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/PreProcessFilter.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/PreProcessFilter.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ReferenceTest.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ReferenceTest.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ScoreArray.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ScoreArray.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ScoreData.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ScoreData.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ScoreDataIterator.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ScoreDataIterator.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ScoreStats.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ScoreStats.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/Scorer.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/Scorer.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ScorerFactory.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ScorerFactory.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/SemposOverlapping.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/SemposOverlapping.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/SemposScorer.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/SemposScorer.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/SentenceLevelScorer.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/SentenceLevelScorer.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/SingletonTest.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/SingletonTest.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/StatisticsBasedScorer.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/StatisticsBasedScorer.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TER + 2 + virtual:/virtual + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TerScorer.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TerScorer.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ThreadPool.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ThreadPool.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/Timer.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/Timer.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TimerTest.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TimerTest.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/Util.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/Util.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/UtilTest.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/UtilTest.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/Vocabulary.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/Vocabulary.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/VocabularyTest.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/VocabularyTest.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/bleu_scorer_test + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/bleu_scorer_test + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/bleu_scorer_test.passed + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/bleu_scorer_test.passed + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/data_test + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/data_test + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/data_test.passed + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/data_test.passed + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/evaluator + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/evaluator + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/evaluator.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/evaluator.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/extractor + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/extractor + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/extractor.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/extractor.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/feature_data_test + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/feature_data_test + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/feature_data_test.passed + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/feature_data_test.passed + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/kbmira + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/kbmira + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/kbmira.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/kbmira.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/libmert_lib.a + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/libmert_lib.a + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/mert + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/mert + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/mert.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/mert.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ngram_test + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ngram_test + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ngram_test.passed + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ngram_test.passed + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/optimizer_factory_test + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/optimizer_factory_test + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/optimizer_factory_test.passed + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/optimizer_factory_test.passed + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/point_test + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/point_test + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/point_test.passed + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/point_test.passed + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/pro + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/pro + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/pro.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/pro.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/reference_test + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/reference_test + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/reference_test.passed + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/reference_test.passed + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/sentence-bleu + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/sentence-bleu + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/sentence-bleu.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/sentence-bleu.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/singleton_test + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/singleton_test + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/singleton_test.passed + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/singleton_test.passed + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/timer_test + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/timer_test + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/timer_test.passed + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/timer_test.passed + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/util_test + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/util_test + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/util_test.passed + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/util_test.passed + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/vocabulary_test + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/vocabulary_test + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/vocabulary_test.passed + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/vocabulary_test.passed + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TER/alignmentStruct.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TER/alignmentStruct.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TER/hashMap.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TER/hashMap.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TER/hashMapInfos.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TER/hashMapInfos.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TER/hashMapStringInfos.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TER/hashMapStringInfos.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TER/infosHasher.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TER/infosHasher.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TER/stringHasher.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TER/stringHasher.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TER/stringInfosHasher.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TER/stringInfosHasher.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TER/terAlignment.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TER/terAlignment.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TER/terShift.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TER/terShift.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TER/tercalc.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TER/tercalc.o + + + bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TER/tools.o + 1 + PARENT-3-PROJECT_LOC/mert/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/TER/tools.o + + + diff --git a/mosesdecoder/contrib/other-builds/lm/lm.project b/mosesdecoder/contrib/other-builds/lm/lm.project new file mode 100644 index 0000000000000000000000000000000000000000..c30ebe533c5df8da201b54ecf00fe15a1c499563 --- /dev/null +++ b/mosesdecoder/contrib/other-builds/lm/lm.project @@ -0,0 +1,140 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mosesdecoder/contrib/other-builds/moses-cmd/.cproject b/mosesdecoder/contrib/other-builds/moses-cmd/.cproject new file mode 100644 index 0000000000000000000000000000000000000000..2dde393bca46f8db29f461ca2e0298a2e49edf9d --- /dev/null +++ b/mosesdecoder/contrib/other-builds/moses-cmd/.cproject @@ -0,0 +1,187 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mosesdecoder/contrib/other-builds/moses-cmd/.project b/mosesdecoder/contrib/other-builds/moses-cmd/.project new file mode 100644 index 0000000000000000000000000000000000000000..9aca6fe7a4dcfc9bac119e29c8dee120f3ceba64 --- /dev/null +++ b/mosesdecoder/contrib/other-builds/moses-cmd/.project @@ -0,0 +1,111 @@ + + + moses-cmd + + + lm + moses + OnDiskPt + probingpt + search + util + + + + org.eclipse.cdt.managedbuilder.core.genmakebuilder + clean,full,incremental, + + + ?name? + + + + org.eclipse.cdt.make.core.append_environment + true + + + org.eclipse.cdt.make.core.autoBuildTarget + all + + + org.eclipse.cdt.make.core.buildArguments + -j3 + + + org.eclipse.cdt.make.core.buildCommand + make + + + org.eclipse.cdt.make.core.buildLocation + ${workspace_loc:/moses-cmd/Debug} + + + org.eclipse.cdt.make.core.cleanBuildTarget + clean + + + org.eclipse.cdt.make.core.contents + org.eclipse.cdt.make.core.activeConfigSettings + + + org.eclipse.cdt.make.core.enableAutoBuild + false + + + org.eclipse.cdt.make.core.enableCleanBuild + true + + + org.eclipse.cdt.make.core.enableFullBuild + true + + + org.eclipse.cdt.make.core.fullBuildTarget + all + + + org.eclipse.cdt.make.core.stopOnError + true + + + org.eclipse.cdt.make.core.useDefaultBuildCmd + true + + + + + org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder + full,incremental, + + + + + + org.eclipse.cdt.core.cnature + org.eclipse.cdt.core.ccnature + org.eclipse.cdt.managedbuilder.core.managedBuildNature + org.eclipse.cdt.managedbuilder.core.ScannerConfigNature + + + + Jamfile + 1 + PARENT-3-PROJECT_LOC/moses-cmd/Jamfile + + + LatticeMBRGrid.cpp + 1 + PARENT-3-PROJECT_LOC/moses-cmd/LatticeMBRGrid.cpp + + + Main.cpp + 1 + PARENT-3-PROJECT_LOC/moses-cmd/Main.cpp + + + Main.h + 1 + PARENT-3-PROJECT_LOC/moses-cmd/Main.h + + + diff --git a/mosesdecoder/contrib/other-builds/moses-cmd/moses-cmd.project b/mosesdecoder/contrib/other-builds/moses-cmd/moses-cmd.project new file mode 100644 index 0000000000000000000000000000000000000000..5303ba7c7febdd18c6ff0ca809a332367cdc9c50 --- /dev/null +++ b/mosesdecoder/contrib/other-builds/moses-cmd/moses-cmd.project @@ -0,0 +1,153 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + None + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + None + + + + + + + + + + + + + + + diff --git a/mosesdecoder/contrib/other-builds/moses2-cmd/.project b/mosesdecoder/contrib/other-builds/moses2-cmd/.project new file mode 100644 index 0000000000000000000000000000000000000000..7b1b96eccac1895defd9155113f71959493fc66b --- /dev/null +++ b/mosesdecoder/contrib/other-builds/moses2-cmd/.project @@ -0,0 +1,44 @@ + + + moses2-cmd + + + lm + moses + moses2 + probingpt + util + + + + org.eclipse.cdt.managedbuilder.core.genmakebuilder + clean,full,incremental, + + + + + org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder + full,incremental, + + + + + + org.eclipse.cdt.core.cnature + org.eclipse.cdt.core.ccnature + org.eclipse.cdt.managedbuilder.core.managedBuildNature + org.eclipse.cdt.managedbuilder.core.ScannerConfigNature + + + + Main.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/Main.cpp + + + Main.h + 1 + PARENT-3-PROJECT_LOC/moses2/Main.h + + + diff --git a/mosesdecoder/contrib/other-builds/moses2/.cproject b/mosesdecoder/contrib/other-builds/moses2/.cproject new file mode 100644 index 0000000000000000000000000000000000000000..1e4a32e1d156d4f74275e1f7d880ce7da0e3d7a2 --- /dev/null +++ b/mosesdecoder/contrib/other-builds/moses2/.cproject @@ -0,0 +1,145 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mosesdecoder/contrib/other-builds/moses2/.project b/mosesdecoder/contrib/other-builds/moses2/.project new file mode 100644 index 0000000000000000000000000000000000000000..1588c88b62322fdeeb7c496b1b7d26b94e7945ee --- /dev/null +++ b/mosesdecoder/contrib/other-builds/moses2/.project @@ -0,0 +1,1621 @@ + + + moses2 + + + moses + util + + + + org.eclipse.cdt.managedbuilder.core.genmakebuilder + clean,full,incremental, + + + + + org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder + full,incremental, + + + + + + org.eclipse.cdt.core.cnature + org.eclipse.cdt.core.ccnature + org.eclipse.cdt.managedbuilder.core.managedBuildNature + org.eclipse.cdt.managedbuilder.core.ScannerConfigNature + + + + AlignmentInfo.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/AlignmentInfo.cpp + + + AlignmentInfo.h + 1 + PARENT-3-PROJECT_LOC/moses2/AlignmentInfo.h + + + AlignmentInfoCollection.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/AlignmentInfoCollection.cpp + + + AlignmentInfoCollection.h + 1 + PARENT-3-PROJECT_LOC/moses2/AlignmentInfoCollection.h + + + ArcLists.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/ArcLists.cpp + + + ArcLists.h + 1 + PARENT-3-PROJECT_LOC/moses2/ArcLists.h + + + Array.h + 1 + PARENT-3-PROJECT_LOC/moses2/Array.h + + + EstimatedScores.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/EstimatedScores.cpp + + + EstimatedScores.h + 1 + PARENT-3-PROJECT_LOC/moses2/EstimatedScores.h + + + FF + 2 + virtual:/virtual + + + HypothesisBase.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/HypothesisBase.cpp + + + HypothesisBase.h + 1 + PARENT-3-PROJECT_LOC/moses2/HypothesisBase.h + + + HypothesisColl.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/HypothesisColl.cpp + + + HypothesisColl.h + 1 + PARENT-3-PROJECT_LOC/moses2/HypothesisColl.h + + + InMemoryTrie + 2 + virtual:/virtual + + + InputPathBase.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/InputPathBase.cpp + + + InputPathBase.h + 1 + PARENT-3-PROJECT_LOC/moses2/InputPathBase.h + + + InputPathsBase.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/InputPathsBase.cpp + + + InputPathsBase.h + 1 + PARENT-3-PROJECT_LOC/moses2/InputPathsBase.h + + + InputType.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/InputType.cpp + + + InputType.h + 1 + PARENT-3-PROJECT_LOC/moses2/InputType.h + + + Jamfile + 1 + PARENT-3-PROJECT_LOC/moses2/Jamfile + + + LM + 2 + virtual:/virtual + + + Main.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/Main.cpp + + + ManagerBase.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/ManagerBase.cpp + + + ManagerBase.h + 1 + PARENT-3-PROJECT_LOC/moses2/ManagerBase.h + + + MemPool.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/MemPool.cpp + + + MemPool.h + 1 + PARENT-3-PROJECT_LOC/moses2/MemPool.h + + + MemPoolAllocator.h + 1 + PARENT-3-PROJECT_LOC/moses2/MemPoolAllocator.h + + + Phrase.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/Phrase.cpp + + + Phrase.h + 1 + PARENT-3-PROJECT_LOC/moses2/Phrase.h + + + PhraseBased + 2 + virtual:/virtual + + + PhraseImplTemplate.h + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseImplTemplate.h + + + Recycler.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/Recycler.cpp + + + Recycler.h + 1 + PARENT-3-PROJECT_LOC/moses2/Recycler.h + + + SCFG + 2 + virtual:/virtual + + + Scores.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/Scores.cpp + + + Scores.h + 1 + PARENT-3-PROJECT_LOC/moses2/Scores.h + + + SubPhrase.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/SubPhrase.cpp + + + SubPhrase.h + 1 + PARENT-3-PROJECT_LOC/moses2/SubPhrase.h + + + System.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/System.cpp + + + System.h + 1 + PARENT-3-PROJECT_LOC/moses2/System.h + + + TargetPhrase.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/TargetPhrase.cpp + + + TargetPhrase.h + 1 + PARENT-3-PROJECT_LOC/moses2/TargetPhrase.h + + + TranslationModel + 2 + virtual:/virtual + + + TranslationTask.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationTask.cpp + + + TranslationTask.h + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationTask.h + + + TrellisPaths.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/TrellisPaths.cpp + + + TrellisPaths.h + 1 + PARENT-3-PROJECT_LOC/moses2/TrellisPaths.h + + + TypeDef.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/TypeDef.cpp + + + TypeDef.h + 1 + PARENT-3-PROJECT_LOC/moses2/TypeDef.h + + + Vector.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/Vector.cpp + + + Vector.h + 1 + PARENT-3-PROJECT_LOC/moses2/Vector.h + + + Weights.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/Weights.cpp + + + Weights.h + 1 + PARENT-3-PROJECT_LOC/moses2/Weights.h + + + Word.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/Word.cpp + + + Word.h + 1 + PARENT-3-PROJECT_LOC/moses2/Word.h + + + defer + 2 + virtual:/virtual + + + legacy + 2 + virtual:/virtual + + + parameters + 2 + virtual:/virtual + + + pugiconfig.hpp + 1 + PARENT-3-PROJECT_LOC/moses2/pugiconfig.hpp + + + pugixml.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/pugixml.cpp + + + pugixml.hpp + 1 + PARENT-3-PROJECT_LOC/moses2/pugixml.hpp + + + server + 2 + virtual:/virtual + + + FF/Distortion.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/FF/Distortion.cpp + + + FF/Distortion.h + 1 + PARENT-3-PROJECT_LOC/moses2/FF/Distortion.h + + + FF/ExampleStatefulFF.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/FF/ExampleStatefulFF.cpp + + + FF/ExampleStatefulFF.h + 1 + PARENT-3-PROJECT_LOC/moses2/FF/ExampleStatefulFF.h + + + FF/ExampleStatelessFF.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/FF/ExampleStatelessFF.cpp + + + FF/ExampleStatelessFF.h + 1 + PARENT-3-PROJECT_LOC/moses2/FF/ExampleStatelessFF.h + + + FF/FFState.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/FF/FFState.cpp + + + FF/FFState.h + 1 + PARENT-3-PROJECT_LOC/moses2/FF/FFState.h + + + FF/FeatureFunction.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/FF/FeatureFunction.cpp + + + FF/FeatureFunction.h + 1 + PARENT-3-PROJECT_LOC/moses2/FF/FeatureFunction.h + + + FF/FeatureFunctions.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/FF/FeatureFunctions.cpp + + + FF/FeatureFunctions.h + 1 + PARENT-3-PROJECT_LOC/moses2/FF/FeatureFunctions.h + + + FF/FeatureRegistry.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/FF/FeatureRegistry.cpp + + + FF/FeatureRegistry.h + 1 + PARENT-3-PROJECT_LOC/moses2/FF/FeatureRegistry.h + + + FF/LexicalReordering + 2 + virtual:/virtual + + + FF/OSM + 2 + virtual:/virtual + + + FF/PhrasePenalty.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/FF/PhrasePenalty.cpp + + + FF/PhrasePenalty.h + 1 + PARENT-3-PROJECT_LOC/moses2/FF/PhrasePenalty.h + + + FF/PointerState.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/FF/PointerState.cpp + + + FF/PointerState.h + 1 + PARENT-3-PROJECT_LOC/moses2/FF/PointerState.h + + + FF/StatefulFeatureFunction.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/FF/StatefulFeatureFunction.cpp + + + FF/StatefulFeatureFunction.h + 1 + PARENT-3-PROJECT_LOC/moses2/FF/StatefulFeatureFunction.h + + + FF/StatelessFeatureFunction.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/FF/StatelessFeatureFunction.cpp + + + FF/StatelessFeatureFunction.h + 1 + PARENT-3-PROJECT_LOC/moses2/FF/StatelessFeatureFunction.h + + + FF/WordPenalty.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/FF/WordPenalty.cpp + + + FF/WordPenalty.h + 1 + PARENT-3-PROJECT_LOC/moses2/FF/WordPenalty.h + + + InMemoryTrie/InMemoryTrie.h + 1 + PARENT-3-PROJECT_LOC/moses2/InMemoryTrie/InMemoryTrie.h + + + InMemoryTrie/Node.h + 1 + PARENT-3-PROJECT_LOC/moses2/InMemoryTrie/Node.h + + + InMemoryTrie/utils.h + 1 + PARENT-3-PROJECT_LOC/moses2/InMemoryTrie/utils.h + + + LM/GPULM.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/LM/GPULM.cpp + + + LM/GPULM.h + 1 + PARENT-3-PROJECT_LOC/moses2/LM/GPULM.h + + + LM/KENLM.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/LM/KENLM.cpp + + + LM/KENLM.h + 1 + PARENT-3-PROJECT_LOC/moses2/LM/KENLM.h + + + LM/KENLMBatch.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/LM/KENLMBatch.cpp + + + LM/KENLMBatch.h + 1 + PARENT-3-PROJECT_LOC/moses2/LM/KENLMBatch.h + + + LM/LanguageModel.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/LM/LanguageModel.cpp + + + LM/LanguageModel.h + 1 + PARENT-3-PROJECT_LOC/moses2/LM/LanguageModel.h + + + PhraseBased/CubePruningMiniStack + 2 + virtual:/virtual + + + PhraseBased/Hypothesis.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/Hypothesis.cpp + + + PhraseBased/Hypothesis.h + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/Hypothesis.h + + + PhraseBased/InputPath.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/InputPath.cpp + + + PhraseBased/InputPath.h + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/InputPath.h + + + PhraseBased/InputPaths.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/InputPaths.cpp + + + PhraseBased/InputPaths.h + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/InputPaths.h + + + PhraseBased/Manager.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/Manager.cpp + + + PhraseBased/Manager.h + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/Manager.h + + + PhraseBased/Normal + 2 + virtual:/virtual + + + PhraseBased/PhraseImpl.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/PhraseImpl.cpp + + + PhraseBased/PhraseImpl.h + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/PhraseImpl.h + + + PhraseBased/ReorderingConstraint.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/ReorderingConstraint.cpp + + + PhraseBased/ReorderingConstraint.h + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/ReorderingConstraint.h + + + PhraseBased/Search.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/Search.cpp + + + PhraseBased/Search.h + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/Search.h + + + PhraseBased/Sentence.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/Sentence.cpp + + + PhraseBased/Sentence.h + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/Sentence.h + + + PhraseBased/TargetPhraseImpl.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/TargetPhraseImpl.cpp + + + PhraseBased/TargetPhraseImpl.h + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/TargetPhraseImpl.h + + + PhraseBased/TargetPhrases.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/TargetPhrases.cpp + + + PhraseBased/TargetPhrases.h + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/TargetPhrases.h + + + PhraseBased/TrellisPath.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/TrellisPath.cpp + + + PhraseBased/TrellisPath.h + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/TrellisPath.h + + + SCFG/ActiveChart.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/ActiveChart.cpp + + + SCFG/ActiveChart.h + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/ActiveChart.h + + + SCFG/Hypothesis.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/Hypothesis.cpp + + + SCFG/Hypothesis.h + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/Hypothesis.h + + + SCFG/InputPath.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/InputPath.cpp + + + SCFG/InputPath.h + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/InputPath.h + + + SCFG/InputPaths.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/InputPaths.cpp + + + SCFG/InputPaths.h + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/InputPaths.h + + + SCFG/Manager.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/Manager.cpp + + + SCFG/Manager.h + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/Manager.h + + + SCFG/Misc.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/Misc.cpp + + + SCFG/Misc.h + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/Misc.h + + + SCFG/PhraseImpl.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/PhraseImpl.cpp + + + SCFG/PhraseImpl.h + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/PhraseImpl.h + + + SCFG/Sentence.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/Sentence.cpp + + + SCFG/Sentence.h + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/Sentence.h + + + SCFG/Stack.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/Stack.cpp + + + SCFG/Stack.h + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/Stack.h + + + SCFG/Stacks.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/Stacks.cpp + + + SCFG/Stacks.h + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/Stacks.h + + + SCFG/TargetPhraseImpl.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/TargetPhraseImpl.cpp + + + SCFG/TargetPhraseImpl.h + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/TargetPhraseImpl.h + + + SCFG/TargetPhrases.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/TargetPhrases.cpp + + + SCFG/TargetPhrases.h + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/TargetPhrases.h + + + SCFG/Word.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/Word.cpp + + + SCFG/Word.h + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/Word.h + + + SCFG/nbest + 2 + virtual:/virtual + + + TranslationModel/CompactPT + 2 + virtual:/virtual + + + TranslationModel/Memory + 2 + virtual:/virtual + + + TranslationModel/PhraseTable.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/PhraseTable.cpp + + + TranslationModel/PhraseTable.h + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/PhraseTable.h + + + TranslationModel/ProbingPT.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/ProbingPT.cpp + + + TranslationModel/ProbingPT.h + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/ProbingPT.h + + + TranslationModel/Transliteration.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/Transliteration.cpp + + + TranslationModel/Transliteration.h + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/Transliteration.h + + + TranslationModel/UnknownWordPenalty.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/UnknownWordPenalty.cpp + + + TranslationModel/UnknownWordPenalty.h + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/UnknownWordPenalty.h + + + defer/CubePruningBitmapStack + 2 + virtual:/virtual + + + defer/CubePruningCardinalStack + 2 + virtual:/virtual + + + defer/CubePruningPerBitmap + 2 + virtual:/virtual + + + defer/CubePruningPerMiniStack + 2 + virtual:/virtual + + + legacy/Bitmap.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/Bitmap.cpp + + + legacy/Bitmap.h + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/Bitmap.h + + + legacy/Bitmaps.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/Bitmaps.cpp + + + legacy/Bitmaps.h + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/Bitmaps.h + + + legacy/Factor.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/Factor.cpp + + + legacy/Factor.h + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/Factor.h + + + legacy/FactorCollection.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/FactorCollection.cpp + + + legacy/FactorCollection.h + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/FactorCollection.h + + + legacy/InputFileStream.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/InputFileStream.cpp + + + legacy/InputFileStream.h + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/InputFileStream.h + + + legacy/Matrix.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/Matrix.cpp + + + legacy/Matrix.h + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/Matrix.h + + + legacy/OutputCollector.h + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/OutputCollector.h + + + legacy/OutputFileStream.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/OutputFileStream.cpp + + + legacy/OutputFileStream.h + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/OutputFileStream.h + + + legacy/Parameter.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/Parameter.cpp + + + legacy/Parameter.h + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/Parameter.h + + + legacy/Range.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/Range.cpp + + + legacy/Range.h + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/Range.h + + + legacy/ThreadPool.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/ThreadPool.cpp + + + legacy/ThreadPool.h + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/ThreadPool.h + + + legacy/Timer.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/Timer.cpp + + + legacy/Timer.h + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/Timer.h + + + legacy/Util2.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/Util2.cpp + + + legacy/Util2.h + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/Util2.h + + + legacy/gzfilebuf.h + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/gzfilebuf.h + + + legacy/xmlrpc-c.h + 1 + PARENT-3-PROJECT_LOC/moses2/legacy/xmlrpc-c.h + + + parameters/AllOptions.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/AllOptions.cpp + + + parameters/AllOptions.h + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/AllOptions.h + + + parameters/BeamSearchOptions.h + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/BeamSearchOptions.h + + + parameters/BookkeepingOptions.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/BookkeepingOptions.cpp + + + parameters/BookkeepingOptions.h + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/BookkeepingOptions.h + + + parameters/ContextParameters.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/ContextParameters.cpp + + + parameters/ContextParameters.h + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/ContextParameters.h + + + parameters/CubePruningOptions.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/CubePruningOptions.cpp + + + parameters/CubePruningOptions.h + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/CubePruningOptions.h + + + parameters/InputOptions.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/InputOptions.cpp + + + parameters/InputOptions.h + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/InputOptions.h + + + parameters/LMBR_Options.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/LMBR_Options.cpp + + + parameters/LMBR_Options.h + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/LMBR_Options.h + + + parameters/LookupOptions.h + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/LookupOptions.h + + + parameters/MBR_Options.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/MBR_Options.cpp + + + parameters/MBR_Options.h + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/MBR_Options.h + + + parameters/NBestOptions.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/NBestOptions.cpp + + + parameters/NBestOptions.h + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/NBestOptions.h + + + parameters/OOVHandlingOptions.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/OOVHandlingOptions.cpp + + + parameters/OOVHandlingOptions.h + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/OOVHandlingOptions.h + + + parameters/OptionsBaseClass.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/OptionsBaseClass.cpp + + + parameters/OptionsBaseClass.h + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/OptionsBaseClass.h + + + parameters/ReorderingOptions.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/ReorderingOptions.cpp + + + parameters/ReorderingOptions.h + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/ReorderingOptions.h + + + parameters/ReportingOptions.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/ReportingOptions.cpp + + + parameters/ReportingOptions.h + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/ReportingOptions.h + + + parameters/SearchOptions.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/SearchOptions.cpp + + + parameters/SearchOptions.h + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/SearchOptions.h + + + parameters/ServerOptions.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/ServerOptions.cpp + + + parameters/ServerOptions.h + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/ServerOptions.h + + + parameters/SyntaxOptions.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/SyntaxOptions.cpp + + + parameters/SyntaxOptions.h + 1 + PARENT-3-PROJECT_LOC/moses2/parameters/SyntaxOptions.h + + + server/Server.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/server/Server.cpp + + + server/Server.h + 1 + PARENT-3-PROJECT_LOC/moses2/server/Server.h + + + server/TranslationRequest.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/server/TranslationRequest.cpp + + + server/TranslationRequest.h + 1 + PARENT-3-PROJECT_LOC/moses2/server/TranslationRequest.h + + + server/Translator.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/server/Translator.cpp + + + server/Translator.h + 1 + PARENT-3-PROJECT_LOC/moses2/server/Translator.h + + + FF/LexicalReordering/BidirectionalReorderingState.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/FF/LexicalReordering/BidirectionalReorderingState.cpp + + + FF/LexicalReordering/BidirectionalReorderingState.h + 1 + PARENT-3-PROJECT_LOC/moses2/FF/LexicalReordering/BidirectionalReorderingState.h + + + FF/LexicalReordering/HReorderingBackwardState.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/FF/LexicalReordering/HReorderingBackwardState.cpp + + + FF/LexicalReordering/HReorderingBackwardState.h + 1 + PARENT-3-PROJECT_LOC/moses2/FF/LexicalReordering/HReorderingBackwardState.h + + + FF/LexicalReordering/HReorderingForwardState.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/FF/LexicalReordering/HReorderingForwardState.cpp + + + FF/LexicalReordering/HReorderingForwardState.h + 1 + PARENT-3-PROJECT_LOC/moses2/FF/LexicalReordering/HReorderingForwardState.h + + + FF/LexicalReordering/LRModel.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/FF/LexicalReordering/LRModel.cpp + + + FF/LexicalReordering/LRModel.h + 1 + PARENT-3-PROJECT_LOC/moses2/FF/LexicalReordering/LRModel.h + + + FF/LexicalReordering/LRState.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/FF/LexicalReordering/LRState.cpp + + + FF/LexicalReordering/LRState.h + 1 + PARENT-3-PROJECT_LOC/moses2/FF/LexicalReordering/LRState.h + + + FF/LexicalReordering/LexicalReordering.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/FF/LexicalReordering/LexicalReordering.cpp + + + FF/LexicalReordering/LexicalReordering.h + 1 + PARENT-3-PROJECT_LOC/moses2/FF/LexicalReordering/LexicalReordering.h + + + FF/LexicalReordering/PhraseBasedReorderingState.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/FF/LexicalReordering/PhraseBasedReorderingState.cpp + + + FF/LexicalReordering/PhraseBasedReorderingState.h + 1 + PARENT-3-PROJECT_LOC/moses2/FF/LexicalReordering/PhraseBasedReorderingState.h + + + FF/LexicalReordering/ReorderingStack.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/FF/LexicalReordering/ReorderingStack.cpp + + + FF/LexicalReordering/ReorderingStack.h + 1 + PARENT-3-PROJECT_LOC/moses2/FF/LexicalReordering/ReorderingStack.h + + + FF/OSM/KenOSM.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/FF/OSM/KenOSM.cpp + + + FF/OSM/KenOSM.h + 1 + PARENT-3-PROJECT_LOC/moses2/FF/OSM/KenOSM.h + + + FF/OSM/OpSequenceModel.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/FF/OSM/OpSequenceModel.cpp + + + FF/OSM/OpSequenceModel.h + 1 + PARENT-3-PROJECT_LOC/moses2/FF/OSM/OpSequenceModel.h + + + FF/OSM/osmHyp.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/FF/OSM/osmHyp.cpp + + + FF/OSM/osmHyp.h + 1 + PARENT-3-PROJECT_LOC/moses2/FF/OSM/osmHyp.h + + + PhraseBased/CubePruningMiniStack/Misc.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/CubePruningMiniStack/Misc.cpp + + + PhraseBased/CubePruningMiniStack/Misc.h + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/CubePruningMiniStack/Misc.h + + + PhraseBased/CubePruningMiniStack/Search.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/CubePruningMiniStack/Search.cpp + + + PhraseBased/CubePruningMiniStack/Search.h + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/CubePruningMiniStack/Search.h + + + PhraseBased/CubePruningMiniStack/Stack.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/CubePruningMiniStack/Stack.cpp + + + PhraseBased/CubePruningMiniStack/Stack.h + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/CubePruningMiniStack/Stack.h + + + PhraseBased/Normal/Search.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/Normal/Search.cpp + + + PhraseBased/Normal/Search.h + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/Normal/Search.h + + + PhraseBased/Normal/Stack.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/Normal/Stack.cpp + + + PhraseBased/Normal/Stack.h + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/Normal/Stack.h + + + PhraseBased/Normal/Stacks.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/Normal/Stacks.cpp + + + PhraseBased/Normal/Stacks.h + 1 + PARENT-3-PROJECT_LOC/moses2/PhraseBased/Normal/Stacks.h + + + SCFG/nbest/KBestExtractor.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/nbest/KBestExtractor.cpp + + + SCFG/nbest/KBestExtractor.h + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/nbest/KBestExtractor.h + + + SCFG/nbest/NBest.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/nbest/NBest.cpp + + + SCFG/nbest/NBest.h + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/nbest/NBest.h + + + SCFG/nbest/NBestColl.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/nbest/NBestColl.cpp + + + SCFG/nbest/NBestColl.h + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/nbest/NBestColl.h + + + SCFG/nbest/NBests.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/nbest/NBests.cpp + + + SCFG/nbest/NBests.h + 1 + PARENT-3-PROJECT_LOC/moses2/SCFG/nbest/NBests.h + + + TranslationModel/CompactPT/BlockHashIndex.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/CompactPT/BlockHashIndex.cpp + + + TranslationModel/CompactPT/BlockHashIndex.h + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/CompactPT/BlockHashIndex.h + + + TranslationModel/CompactPT/CanonicalHuffman.h + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/CompactPT/CanonicalHuffman.h + + + TranslationModel/CompactPT/CmphStringVectorAdapter.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/CompactPT/CmphStringVectorAdapter.cpp + + + TranslationModel/CompactPT/CmphStringVectorAdapter.h + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/CompactPT/CmphStringVectorAdapter.h + + + TranslationModel/CompactPT/LexicalReorderingTableCompact.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/CompactPT/LexicalReorderingTableCompact.cpp + + + TranslationModel/CompactPT/LexicalReorderingTableCompact.h + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/CompactPT/LexicalReorderingTableCompact.h + + + TranslationModel/CompactPT/ListCoders.h + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/CompactPT/ListCoders.h + + + TranslationModel/CompactPT/MmapAllocator.h + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/CompactPT/MmapAllocator.h + + + TranslationModel/CompactPT/MonotonicVector.h + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/CompactPT/MonotonicVector.h + + + TranslationModel/CompactPT/MurmurHash3.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/CompactPT/MurmurHash3.cpp + + + TranslationModel/CompactPT/MurmurHash3.h + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/CompactPT/MurmurHash3.h + + + TranslationModel/CompactPT/PackedArray.h + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/CompactPT/PackedArray.h + + + TranslationModel/CompactPT/StringVector.h + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/CompactPT/StringVector.h + + + TranslationModel/CompactPT/TargetPhraseCollectionCache.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/CompactPT/TargetPhraseCollectionCache.cpp + + + TranslationModel/CompactPT/TargetPhraseCollectionCache.h + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/CompactPT/TargetPhraseCollectionCache.h + + + TranslationModel/CompactPT/ThrowingFwrite.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/CompactPT/ThrowingFwrite.cpp + + + TranslationModel/CompactPT/ThrowingFwrite.h + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/CompactPT/ThrowingFwrite.h + + + TranslationModel/Memory/Node.h + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/Memory/Node.h + + + TranslationModel/Memory/PhraseTableMemory.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/Memory/PhraseTableMemory.cpp + + + TranslationModel/Memory/PhraseTableMemory.h + 1 + PARENT-3-PROJECT_LOC/moses2/TranslationModel/Memory/PhraseTableMemory.h + + + defer/CubePruningBitmapStack/Misc.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningBitmapStack/Misc.cpp + + + defer/CubePruningBitmapStack/Misc.h + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningBitmapStack/Misc.h + + + defer/CubePruningBitmapStack/Search.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningBitmapStack/Search.cpp + + + defer/CubePruningBitmapStack/Search.h + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningBitmapStack/Search.h + + + defer/CubePruningBitmapStack/Stack.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningBitmapStack/Stack.cpp + + + defer/CubePruningBitmapStack/Stack.h + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningBitmapStack/Stack.h + + + defer/CubePruningCardinalStack/Misc.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningCardinalStack/Misc.cpp + + + defer/CubePruningCardinalStack/Misc.h + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningCardinalStack/Misc.h + + + defer/CubePruningCardinalStack/Search.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningCardinalStack/Search.cpp + + + defer/CubePruningCardinalStack/Search.h + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningCardinalStack/Search.h + + + defer/CubePruningCardinalStack/Stack.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningCardinalStack/Stack.cpp + + + defer/CubePruningCardinalStack/Stack.h + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningCardinalStack/Stack.h + + + defer/CubePruningPerBitmap/Misc.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningPerBitmap/Misc.cpp + + + defer/CubePruningPerBitmap/Misc.h + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningPerBitmap/Misc.h + + + defer/CubePruningPerBitmap/Search.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningPerBitmap/Search.cpp + + + defer/CubePruningPerBitmap/Search.h + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningPerBitmap/Search.h + + + defer/CubePruningPerBitmap/Stacks.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningPerBitmap/Stacks.cpp + + + defer/CubePruningPerBitmap/Stacks.h + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningPerBitmap/Stacks.h + + + defer/CubePruningPerMiniStack/Misc.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningPerMiniStack/Misc.cpp + + + defer/CubePruningPerMiniStack/Misc.h + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningPerMiniStack/Misc.h + + + defer/CubePruningPerMiniStack/Search.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningPerMiniStack/Search.cpp + + + defer/CubePruningPerMiniStack/Search.h + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningPerMiniStack/Search.h + + + defer/CubePruningPerMiniStack/Stacks.cpp + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningPerMiniStack/Stacks.cpp + + + defer/CubePruningPerMiniStack/Stacks.h + 1 + PARENT-3-PROJECT_LOC/moses2/defer/CubePruningPerMiniStack/Stacks.h + + + diff --git a/mosesdecoder/contrib/other-builds/moses2/moses2.sln b/mosesdecoder/contrib/other-builds/moses2/moses2.sln new file mode 100644 index 0000000000000000000000000000000000000000..053c000df4f1a1f3b4f536b9828c956628c95525 --- /dev/null +++ b/mosesdecoder/contrib/other-builds/moses2/moses2.sln @@ -0,0 +1,43 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.9.34607.119 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "moses2", "moses2.vcxproj", "{B4304E97-D37F-4022-BD03-841A4FAEE398}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + Debug-exe|Win32 = Debug-exe|Win32 + Debug-exe|x64 = Debug-exe|x64 + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + Release-exe|Win32 = Release-exe|Win32 + Release-exe|x64 = Release-exe|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {B4304E97-D37F-4022-BD03-841A4FAEE398}.Debug|Win32.ActiveCfg = Debug|Win32 + {B4304E97-D37F-4022-BD03-841A4FAEE398}.Debug|Win32.Build.0 = Debug|Win32 + {B4304E97-D37F-4022-BD03-841A4FAEE398}.Debug|x64.ActiveCfg = Debug|x64 + {B4304E97-D37F-4022-BD03-841A4FAEE398}.Debug|x64.Build.0 = Debug|x64 + {B4304E97-D37F-4022-BD03-841A4FAEE398}.Debug-exe|Win32.ActiveCfg = Debug-exe|x64 + {B4304E97-D37F-4022-BD03-841A4FAEE398}.Debug-exe|Win32.Build.0 = Debug-exe|x64 + {B4304E97-D37F-4022-BD03-841A4FAEE398}.Debug-exe|x64.ActiveCfg = Debug-exe|x64 + {B4304E97-D37F-4022-BD03-841A4FAEE398}.Debug-exe|x64.Build.0 = Debug-exe|x64 + {B4304E97-D37F-4022-BD03-841A4FAEE398}.Release|Win32.ActiveCfg = Release|Win32 + {B4304E97-D37F-4022-BD03-841A4FAEE398}.Release|Win32.Build.0 = Release|Win32 + {B4304E97-D37F-4022-BD03-841A4FAEE398}.Release|x64.ActiveCfg = Release|x64 + {B4304E97-D37F-4022-BD03-841A4FAEE398}.Release|x64.Build.0 = Release|x64 + {B4304E97-D37F-4022-BD03-841A4FAEE398}.Release-exe|Win32.ActiveCfg = Release-exe|x64 + {B4304E97-D37F-4022-BD03-841A4FAEE398}.Release-exe|Win32.Build.0 = Release-exe|x64 + {B4304E97-D37F-4022-BD03-841A4FAEE398}.Release-exe|x64.ActiveCfg = Release-exe|x64 + {B4304E97-D37F-4022-BD03-841A4FAEE398}.Release-exe|x64.Build.0 = Release-exe|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {8B10AABC-E114-4EB7-BE1E-9432E208DDEA} + EndGlobalSection +EndGlobal diff --git a/mosesdecoder/contrib/other-builds/moses2/moses2.vcxproj b/mosesdecoder/contrib/other-builds/moses2/moses2.vcxproj new file mode 100644 index 0000000000000000000000000000000000000000..df1fc34271d03e1ebd2b9f18538ca9a585110bed --- /dev/null +++ b/mosesdecoder/contrib/other-builds/moses2/moses2.vcxproj @@ -0,0 +1,592 @@ + + + + + Debug-exe2 + x64 + + + Debug-exe + x64 + + + Debug + x64 + + + Release-exe + x64 + + + Release + x64 + + + + {B4304E97-D37F-4022-BD03-841A4FAEE398} + Win32Proj + moses2 + 10.0.18362.0 + v140 + c:\depot\translator\packages\boost.library\1.66.0 + c:\depot\translator\private\mt\zlib\src + + + + DynamicLibrary + true + Unicode + false + false + + + Application + true + Unicode + false + false + + + Application + true + Unicode + false + false + + + DynamicLibrary + true + Unicode + false + false + + + DynamicLibrary + false + true + Unicode + + + Application + false + true + Unicode + + + DynamicLibrary + false + true + Unicode + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + $(ProjectName)decoder + + + true + $(ProjectName)decoder + + + true + $(ProjectName)decoder + + + true + $(ProjectName)decoder + + + false + $(ProjectName)decoder + + + false + $(ProjectName)decoder + + + false + $(ProjectName)decoder + + + + + + Level3 + Disabled + NO_COMPACT_TABLES;HAVE_CMPHXXX;MAX_NUM_FACTORS=4;KENLM_MAX_ORDER=6;_USE_MATH_DEFINES;NOMINMAX;WITH_THREADS;NO_PIPES;_WIN32;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + $(ZlibDir);$(BoostDir);$(MSBuildProjectDirectory)/../../..;C:\Program Files (x86)\Visual Leak Detector\include;C:\depot\translator\private\mt\mman;%(AdditionalIncludeDirectories) + MultiThreadedDebugDLL + $(IntDir)\%(RelativeDir)\%(Filename).obj + false + false + Default + + + ProgramDatabase + false + + + true + + + Console + true + $(BoostDir)\lib64-msvc-14.0;$(ZlibDir)\dll\bin\x64\Debug;C:\Program Files (x86)\Visual Leak Detector\lib\Win64 + %(AdditionalDependencies) + LinkVerbose + false + + + + + + + Level3 + Disabled + NO_COMPACT_TABLES;HAVE_CMPHXXX;MAX_NUM_FACTORS=4;KENLM_MAX_ORDER=6;_USE_MATH_DEFINES;NOMINMAX;WITH_THREADS;NO_PIPES;_WIN32;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + $(ZlibDir);$(BoostDir);$(MSBuildProjectDirectory)/../../..;C:\Program Files (x86)\Visual Leak Detector\include;C:\depot\translator\private\mt\mman;%(AdditionalIncludeDirectories) + MultiThreadedDebugDLL + $(IntDir)\%(RelativeDir)\%(Filename).obj + false + false + Default + + + ProgramDatabase + false + + + true + + + Console + true + $(BoostDir)\lib64-msvc-14.0;$(ZlibDir)\dll\bin\x64\Debug;C:\Program Files (x86)\Visual Leak Detector\lib\Win64 + %(AdditionalDependencies) + LinkVerbose + false + + + + + + + Level3 + Disabled + NO_COMPACT_TABLES;HAVE_CMPHXXX;MAX_NUM_FACTORS=4;KENLM_MAX_ORDER=6;_USE_MATH_DEFINES;NOMINMAX;WITH_THREADS;NO_PIPES;_WIN32;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + $(ZlibDir);$(BoostDir);$(MSBuildProjectDirectory)/../../..;C:\Program Files (x86)\Visual Leak Detector\include;C:\depot\translator\private\mt\mman;%(AdditionalIncludeDirectories) + MultiThreadedDebugDLL + $(IntDir)\%(RelativeDir)\%(Filename).obj + false + false + Default + + + ProgramDatabase + false + + + true + + + Console + true + $(BoostDir)\lib64-msvc-14.0;$(ZlibDir)\dll\bin\x64\Debug;C:\Program Files (x86)\Visual Leak Detector\lib\Win64 + %(AdditionalDependencies) + LinkVerbose + false + + + + + + + Level3 + Disabled + NO_COMPACT_TABLES;HAVE_CMPHXXX;MAX_NUM_FACTORS=4;KENLM_MAX_ORDER=6;_USE_MATH_DEFINES;NOMINMAX;WITH_THREADS;NO_PIPES;_WIN32;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + $(ZlibDir);$(BoostDir);$(MSBuildProjectDirectory)/../../..;C:\Program Files (x86)\Visual Leak Detector\include;C:\depot\translator\private\mt\mman;%(AdditionalIncludeDirectories) + MultiThreadedDebugDLL + $(IntDir)\%(RelativeDir)\%(Filename).obj + false + false + Default + + + ProgramDatabase + false + + + true + + + Console + true + $(BoostDir)\lib64-msvc-14.0;$(ZlibDir)\dll\bin\x64\Debug;C:\Program Files (x86)\Visual Leak Detector\lib\Win64 + %(AdditionalDependencies) + LinkVerbose + false + + + + + Level3 + + + MaxSpeed + true + true + NO_COMPACT_TABLES;HAVE_CMPHXXX;MAX_NUM_FACTORS=4;KENLM_MAX_ORDER=6;_USE_MATH_DEFINES;NOMINMAX;WITH_THREADS;NO_PIPES;_WIN32;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + $(ZlibDir);$(BoostDir);$(MSBuildProjectDirectory)/../../..;C:\Program Files (x86)\Visual Leak Detector\include;C:\depot\translator\private\mt\mman;%(AdditionalIncludeDirectories) + MultiThreadedDLL + $(IntDir)\%(RelativeDir)\%(Filename).obj + false + Default + ProgramDatabase + true + + + Console + true + true + true + $(BoostDir)\lib64-msvc-14.0;$(ZlibDir)\dll\bin\x64\Debug;C:\Program Files (x86)\Visual Leak Detector\lib\Win64 + %(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + NO_COMPACT_TABLES;HAVE_CMPHXXX;MAX_NUM_FACTORS=4;KENLM_MAX_ORDER=6;_USE_MATH_DEFINES;NOMINMAX;WITH_THREADS;NO_PIPES;_WIN32;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + $(ZlibDir);$(BoostDir);$(MSBuildProjectDirectory)/../../..;C:\Program Files (x86)\Visual Leak Detector\include;C:\depot\translator\private\mt\mman;%(AdditionalIncludeDirectories) + MultiThreadedDLL + $(IntDir)\%(RelativeDir)\%(Filename).obj + false + Default + ProgramDatabase + true + + + Console + true + true + true + $(BoostDir)\lib64-msvc-14.0;$(ZlibDir)\dll\bin\x64\Debug;C:\Program Files (x86)\Visual Leak Detector\lib\Win64 + %(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + NO_COMPACT_TABLES;HAVE_CMPHXXX;MAX_NUM_FACTORS=4;KENLM_MAX_ORDER=6;_USE_MATH_DEFINES;NOMINMAX;WITH_THREADS;NO_PIPES;_WIN32;WIN32;NDEBUG;_CONSOLE;_LIB;NO_PROBING_PT;%(PreprocessorDefinitions) + $(ZlibDir);$(BoostDir);$(MSBuildProjectDirectory)/../../..;%(AdditionalIncludeDirectories) + MultiThreaded + $(IntDir)\%(RelativeDir)\%(Filename).obj + false + Default + ProgramDatabase + true + + + Console + true + true + true + $(BoostDir)\lib64-msvc-14.0;$(ZlibDir)\dll\bin\x64\Debug + %(AdditionalDependencies) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + + + true + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mosesdecoder/contrib/other-builds/moses2/moses2.vcxproj.filters b/mosesdecoder/contrib/other-builds/moses2/moses2.vcxproj.filters new file mode 100644 index 0000000000000000000000000000000000000000..6995b6ada1c13b582f473579e18fc45fccfb8caf --- /dev/null +++ b/mosesdecoder/contrib/other-builds/moses2/moses2.vcxproj.filters @@ -0,0 +1,768 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + {214e915b-eafb-4e76-b9a2-1fbfe99424b7} + + + {d6922dd8-d86c-4c79-8587-a3a412a2e9f6} + + + {4e9f8bc8-ef01-463e-a309-df80bb1e63b5} + + + {58e84bec-0301-41b2-a4c2-dc00162c550a} + + + {ccdac19b-8883-4c3d-8006-365d4846688f} + + + {929b6fab-e56b-4218-bfb6-da1c50a6c48e} + + + {797f4634-c680-45fc-b30e-b5e90ee5d224} + + + {5a603a90-587d-4fef-ab01-ba6e5173869e} + + + {85931b40-138d-48c4-a288-c4ee96039879} + + + {a044c0cd-45cc-4a91-bd83-0e04d11f2afa} + + + {fe76b14f-0997-4f14-b3d6-c6f1b725bf72} + + + {dea1c12e-1a75-4313-9c03-28689a06a1ee} + + + {b895cea0-249f-4b4a-9b47-872d810fd4f2} + + + {12cccf90-d56a-4aca-8780-41777bb2f291} + + + {e8564ac2-1055-4b10-9da0-4ae82e713881} + + + {cad8906a-d5b9-447d-952c-68bd6fa93bef} + + + {806d58ef-f545-4428-a8f0-f870e211d15f} + + + {8f10d9c1-66d4-4490-b310-d3f4973bad29} + + + {8f2f68ba-2b5e-4d2c-92a7-6ece71df74ba} + + + {742ab76a-e9db-4098-b7dd-38a4aeac845e} + + + {4bcc6bd2-7220-4a5a-95e1-fd08b03a0d34} + + + + + Source Files\FF + + + Source Files\FF + + + Source Files\FF + + + Source Files\FF + + + Source Files\FF + + + Source Files\FF + + + Source Files\FF + + + Source Files\FF + + + Source Files\FF + + + Source Files\FF + + + Source Files\FF + + + Source Files\FF + + + Source Files\FF\OSM + + + Source Files\FF\OSM + + + Source Files\FF\OSM + + + Source Files\FF\LexicalReordering + + + Source Files\FF\LexicalReordering + + + Source Files\FF\LexicalReordering + + + Source Files\FF\LexicalReordering + + + Source Files\FF\LexicalReordering + + + Source Files\FF\LexicalReordering + + + Source Files\FF\LexicalReordering + + + Source Files\FF\LexicalReordering + + + Source Files\legacy + + + Source Files\legacy + + + Source Files\legacy + + + Source Files\legacy + + + Source Files\legacy + + + Source Files\legacy + + + Source Files\legacy + + + Source Files\legacy + + + Source Files\legacy + + + Source Files\legacy + + + Source Files\legacy + + + Source Files\legacy + + + Source Files\Moses2LM + + + Source Files\Moses2LM + + + Source Files\Moses2LM + + + Source Files\Moses2LM + + + Source Files\parameters + + + Source Files\parameters + + + Source Files\parameters + + + Source Files\parameters + + + Source Files\parameters + + + Source Files\parameters + + + Source Files\parameters + + + Source Files\parameters + + + Source Files\parameters + + + Source Files\parameters + + + Source Files\parameters + + + Source Files\parameters + + + Source Files\parameters + + + Source Files\parameters + + + Source Files\PhraseBased + + + Source Files\PhraseBased + + + Source Files\PhraseBased + + + Source Files\PhraseBased + + + Source Files\PhraseBased + + + Source Files\PhraseBased + + + Source Files\PhraseBased + + + Source Files\PhraseBased + + + Source Files\PhraseBased + + + Source Files\PhraseBased + + + Source Files\PhraseBased + + + Source Files\PhraseBased + + + Source Files\PhraseBased\Normal + + + Source Files\PhraseBased\Normal + + + Source Files\PhraseBased\Normal + + + Source Files\PhraseBased\CubePruningMiniStack + + + Source Files\PhraseBased\CubePruningMiniStack + + + Source Files\PhraseBased\CubePruningMiniStack + + + Source Files\ProbingPT + + + Source Files\ProbingPT + + + Source Files\ProbingPT + + + Source Files\ProbingPT + + + Source Files\SCFG + + + Source Files\SCFG + + + Source Files\SCFG + + + Source Files\SCFG + + + Source Files\SCFG + + + Source Files\SCFG + + + Source Files\SCFG + + + Source Files\SCFG + + + Source Files\SCFG + + + Source Files\SCFG + + + Source Files\SCFG + + + Source Files\SCFG + + + Source Files\SCFG + + + Source Files\SCFG\nbest + + + Source Files\SCFG\nbest + + + Source Files\SCFG\nbest + + + Source Files\SCFG\nbest + + + Source Files\TranslationModel + + + Source Files\TranslationModel + + + Source Files\TranslationModel + + + Source Files\TranslationModel\Memory + + + Source Files\util\double-conversion + + + Source Files\util\double-conversion + + + Source Files\util\double-conversion + + + Source Files\util\double-conversion + + + Source Files\util\double-conversion + + + Source Files\util\double-conversion + + + Source Files\util\double-conversion + + + Source Files\util\double-conversion + + + Source Files\util + + + Source Files\util + + + Source Files\util + + + Source Files\util + + + Source Files\util + + + Source Files\util + + + Source Files\util + + + Source Files\util + + + Source Files\util + + + Source Files\util + + + Source Files\util + + + Source Files\util + + + Source Files\util + + + Source Files\util + + + Source Files\util + + + Source Files\util + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files\lm + + + Source Files\lm + + + Source Files\lm + + + Source Files\lm + + + Source Files\lm + + + Source Files\lm + + + Source Files\lm + + + Source Files\lm + + + Source Files\lm + + + Source Files\lm + + + Source Files\lm + + + Source Files\lm + + + Source Files\lm + + + Source Files\lm + + + Source Files\lm + + + Source Files + + + Source Files\parameters + + + Source Files\TranslationModel\Dynamic + + + Source Files + + + Source Files\TranslationModel + + + Source Files\ProbingPT + + + Source Files\ProbingPT + + + Source Files\legacy + + + + + + + + Source Files\TranslationModel + + + Source Files\ProbingPT + + + Source Files\ProbingPT + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files\legacy + + + Header Files\legacy + + + Header Files\legacy + + + Header Files\legacy + + + Header Files\legacy + + + Header Files\legacy + + + Header Files\legacy + + + Header Files\legacy + + + Header Files\legacy + + + Header Files\legacy + + + Header Files\legacy + + + Header Files\legacy + + + Header Files\legacy + + + Header Files\legacy + + + Header Files\legacy + + + Header Files\FF + + + Header Files\FF + + + Header Files\FF + + + Header Files\FF + + + Header Files\FF + + + Header Files\FF + + + Header Files\FF + + + Header Files\FF + + + Header Files\FF + + + Header Files\FF + + + Header Files\FF + + + Header Files\FF + + + \ No newline at end of file diff --git a/mosesdecoder/contrib/other-builds/probingpt/.cproject b/mosesdecoder/contrib/other-builds/probingpt/.cproject new file mode 100644 index 0000000000000000000000000000000000000000..155f077a05b038dabcfc0b87da42772216459596 --- /dev/null +++ b/mosesdecoder/contrib/other-builds/probingpt/.cproject @@ -0,0 +1,126 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mosesdecoder/contrib/other-builds/probingpt/.project b/mosesdecoder/contrib/other-builds/probingpt/.project new file mode 100644 index 0000000000000000000000000000000000000000..023a89212a11b36825a7351e0c759591b6861e51 --- /dev/null +++ b/mosesdecoder/contrib/other-builds/probingpt/.project @@ -0,0 +1,34 @@ + + + probingpt + + + + + + org.eclipse.cdt.managedbuilder.core.genmakebuilder + clean,full,incremental, + + + + + org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder + full,incremental, + + + + + + org.eclipse.cdt.core.cnature + org.eclipse.cdt.core.ccnature + org.eclipse.cdt.managedbuilder.core.managedBuildNature + org.eclipse.cdt.managedbuilder.core.ScannerConfigNature + + + + probingpt + 2 + PARENT-3-PROJECT_LOC/probingpt + + + diff --git a/mosesdecoder/contrib/other-builds/score/.cproject b/mosesdecoder/contrib/other-builds/score/.cproject new file mode 100644 index 0000000000000000000000000000000000000000..a6d4e1b8849938d027621aa5578d1cb19d9f8ec4 --- /dev/null +++ b/mosesdecoder/contrib/other-builds/score/.cproject @@ -0,0 +1,153 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mosesdecoder/contrib/other-builds/score/.project b/mosesdecoder/contrib/other-builds/score/.project new file mode 100644 index 0000000000000000000000000000000000000000..10e71312460979a26c47505834b8555fbc1fd0bd --- /dev/null +++ b/mosesdecoder/contrib/other-builds/score/.project @@ -0,0 +1,106 @@ + + + score + + + moses + util + + + + org.eclipse.cdt.managedbuilder.core.genmakebuilder + clean,full,incremental, + + + + + org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder + full,incremental, + + + + + + org.eclipse.cdt.core.cnature + org.eclipse.cdt.core.ccnature + org.eclipse.cdt.managedbuilder.core.managedBuildNature + org.eclipse.cdt.managedbuilder.core.ScannerConfigNature + + + + DomainFeature.cpp + 1 + PARENT-3-PROJECT_LOC/phrase-extract/DomainFeature.cpp + + + DomainFeature.h + 1 + PARENT-3-PROJECT_LOC/phrase-extract/DomainFeature.h + + + ExtractionPhrasePair.cpp + 1 + PARENT-3-PROJECT_LOC/phrase-extract/ExtractionPhrasePair.cpp + + + ExtractionPhrasePair.h + 1 + PARENT-3-PROJECT_LOC/phrase-extract/ExtractionPhrasePair.h + + + InputFileStream.cpp + 1 + PARENT-3-PROJECT_LOC/phrase-extract/InputFileStream.cpp + + + InputFileStream.h + 1 + PARENT-3-PROJECT_LOC/phrase-extract/InputFileStream.h + + + InternalStructFeature.cpp + 1 + PARENT-3-PROJECT_LOC/phrase-extract/InternalStructFeature.cpp + + + InternalStructFeature.h + 1 + PARENT-3-PROJECT_LOC/phrase-extract/InternalStructFeature.h + + + OutputFileStream.cpp + 1 + PARENT-3-PROJECT_LOC/phrase-extract/OutputFileStream.cpp + + + OutputFileStream.h + 1 + PARENT-3-PROJECT_LOC/phrase-extract/OutputFileStream.h + + + ScoreFeature.cpp + 1 + PARENT-3-PROJECT_LOC/phrase-extract/ScoreFeature.cpp + + + ScoreFeature.h + 1 + PARENT-3-PROJECT_LOC/phrase-extract/ScoreFeature.h + + + score-main.cpp + 1 + PARENT-3-PROJECT_LOC/phrase-extract/score-main.cpp + + + tables-core.cpp + 1 + PARENT-3-PROJECT_LOC/phrase-extract/tables-core.cpp + + + tables-core.h + 1 + PARENT-3-PROJECT_LOC/phrase-extract/tables-core.h + + + diff --git a/mosesdecoder/contrib/other-builds/score/score.project b/mosesdecoder/contrib/other-builds/score/score.project new file mode 100644 index 0000000000000000000000000000000000000000..08e0b9414b58e93878c5bc3d15a26526e36c0bce --- /dev/null +++ b/mosesdecoder/contrib/other-builds/score/score.project @@ -0,0 +1,140 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + None + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + None + + + + + + + + + + + + + + + diff --git a/mosesdecoder/contrib/other-builds/search/.cproject b/mosesdecoder/contrib/other-builds/search/.cproject new file mode 100644 index 0000000000000000000000000000000000000000..ad505c5695a397cc6b9f7785bda03ec1fc3be1fa --- /dev/null +++ b/mosesdecoder/contrib/other-builds/search/.cproject @@ -0,0 +1,139 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mosesdecoder/contrib/other-builds/search/.project b/mosesdecoder/contrib/other-builds/search/.project new file mode 100644 index 0000000000000000000000000000000000000000..db756f1d445bf0f711c51332efec0d8296539fd6 --- /dev/null +++ b/mosesdecoder/contrib/other-builds/search/.project @@ -0,0 +1,166 @@ + + + search + + + + + + org.eclipse.cdt.managedbuilder.core.genmakebuilder + clean,full,incremental, + + + ?name? + + + + org.eclipse.cdt.make.core.append_environment + true + + + org.eclipse.cdt.make.core.autoBuildTarget + all + + + org.eclipse.cdt.make.core.buildArguments + + + + org.eclipse.cdt.make.core.buildCommand + make + + + org.eclipse.cdt.make.core.buildLocation + ${workspace_loc:/search/Debug} + + + org.eclipse.cdt.make.core.cleanBuildTarget + clean + + + org.eclipse.cdt.make.core.contents + org.eclipse.cdt.make.core.activeConfigSettings + + + org.eclipse.cdt.make.core.enableAutoBuild + false + + + org.eclipse.cdt.make.core.enableCleanBuild + true + + + org.eclipse.cdt.make.core.enableFullBuild + true + + + org.eclipse.cdt.make.core.fullBuildTarget + all + + + org.eclipse.cdt.make.core.stopOnError + true + + + org.eclipse.cdt.make.core.useDefaultBuildCmd + true + + + + + org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder + full,incremental, + + + + + + org.eclipse.cdt.core.cnature + org.eclipse.cdt.core.ccnature + org.eclipse.cdt.managedbuilder.core.managedBuildNature + org.eclipse.cdt.managedbuilder.core.ScannerConfigNature + + + + Jamfile + 1 + PARENT-3-PROJECT_LOC/search/Jamfile + + + applied.hh + 1 + PARENT-3-PROJECT_LOC/search/applied.hh + + + config.hh + 1 + PARENT-3-PROJECT_LOC/search/config.hh + + + context.hh + 1 + PARENT-3-PROJECT_LOC/search/context.hh + + + edge.hh + 1 + PARENT-3-PROJECT_LOC/search/edge.hh + + + edge_generator.cc + 1 + PARENT-3-PROJECT_LOC/search/edge_generator.cc + + + edge_generator.hh + 1 + PARENT-3-PROJECT_LOC/search/edge_generator.hh + + + header.hh + 1 + PARENT-3-PROJECT_LOC/search/header.hh + + + nbest.cc + 1 + PARENT-3-PROJECT_LOC/search/nbest.cc + + + nbest.hh + 1 + PARENT-3-PROJECT_LOC/search/nbest.hh + + + rule.cc + 1 + PARENT-3-PROJECT_LOC/search/rule.cc + + + rule.hh + 1 + PARENT-3-PROJECT_LOC/search/rule.hh + + + types.hh + 1 + PARENT-3-PROJECT_LOC/search/types.hh + + + vertex.cc + 1 + PARENT-3-PROJECT_LOC/search/vertex.cc + + + vertex.hh + 1 + PARENT-3-PROJECT_LOC/search/vertex.hh + + + vertex_generator.hh + 1 + PARENT-3-PROJECT_LOC/search/vertex_generator.hh + + + + diff --git a/mosesdecoder/contrib/other-builds/search/search.project b/mosesdecoder/contrib/other-builds/search/search.project new file mode 100644 index 0000000000000000000000000000000000000000..8be29fd1d8d615012730c3896286fd77aa184da2 --- /dev/null +++ b/mosesdecoder/contrib/other-builds/search/search.project @@ -0,0 +1,106 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mosesdecoder/contrib/promix/sampler.py b/mosesdecoder/contrib/promix/sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..ffd6976902bf38677e497d1b61caafb69e48c78b --- /dev/null +++ b/mosesdecoder/contrib/promix/sampler.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python + +# +# +# + +import heapq +import math +import random +import sys + +from bleu import BleuScorer + + +class Sample: + """A pair of hypotheses, and their score difference""" + def __init__(self,hyp1,hyp2): + self.hyp1 = hyp1 + self.hyp2 = hyp2 + self.diff = abs(hyp1.score-hyp2.score) + + def __cmp__(self,other): + return cmp(self.diff,other.diff) + +class HopkinsMaySampler: + """Implements Hopkins & May sampling""" + def __init__(self): + self.ncandidates = 5000 # Gamma in Hopkins and May + self.nsamples = 50 # Xi in Hopkins and May + self.min_diff = 0.05 # Minimum scoring difference + + def sample(self,nbest): + samples = [] + for i in xrange(self.ncandidates): + hyp1 = random.choice(nbest.hyps) + hyp2 = random.choice(nbest.hyps) + sample = Sample(hyp1,hyp2) + if sample.diff < self.min_diff: continue + # maintain nsamples biggest samples + heapq.heappush(samples,sample) + while len(samples) > self.nsamples: + heapq.heappop(samples) + return samples + diff --git a/mosesdecoder/contrib/promix/test_bleu.py b/mosesdecoder/contrib/promix/test_bleu.py new file mode 100644 index 0000000000000000000000000000000000000000..28708bcd9359e599783b9fa872cd9031c34983cf --- /dev/null +++ b/mosesdecoder/contrib/promix/test_bleu.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python + +import math +import unittest + +from bleu import * + +class TestBleuScorer(unittest.TestCase): + def setUp(self): + self.scorer = BleuScorer() + + def test_wrong_length(self): + self.assertRaises(BleuScoreException, self.scorer.score, [1]*8) + self.assertRaises(BleuScoreException, self.scorer.score, [1]*10) + + def test_score(self): + stats = [2,5,0,1,1,1,1,1,5] + self.assertAlmostEqual(self.scorer.score(stats), 1/math.sqrt(2)) + + def test_brevity(self): + stats = [2,2,2,2,2,2,2,2,3] + self.assertAlmostEqual(self.scorer.score(stats), math.exp(1 - 3.0/2.0)) + +if __name__ == "__main__": + unittest.main() + +suite = unittest.TestLoader().loadTestsFromTestCase(TestBleuScorer) + diff --git a/mosesdecoder/contrib/promix/train.py b/mosesdecoder/contrib/promix/train.py new file mode 100644 index 0000000000000000000000000000000000000000..2a0391db6ba686a29eb1630b21104802fc0ea645 --- /dev/null +++ b/mosesdecoder/contrib/promix/train.py @@ -0,0 +1,412 @@ +#!/usr/bin/env python + +# +# Train the model weights +# + +from math import log,exp +import sys +import numpy as np +from scipy.optimize.optimize import fmin_cg, fmin_bfgs, fmin +from scipy.optimize.lbfgsb import fmin_l_bfgs_b + +import nbest +from util import safelog + +def sigmoid(x): + return 1.0 / (1.0 + np.exp(-x)) + +class OptimisationException(Exception): + pass + +class ParabaloidOptimiser: + """Optimises a very simple function, to test scipy""" + def __init__(self, params): + self.params = params + + def objective(self,x): + return np.sum(x*x*self.params*self.params) + + def grad_k(self,x,k): + return 2 * self.params[k]**2 * x[k] + + def grad(self,x): + return np.array([self.grad_k(x,k) for k in range(len(x))]) + + def debug(self,x): + print "x = ",x + + + def optimise_bfgs(self,start): + print + print "***** BFGS OPTIMISATION *****" + return fmin_bfgs(self.objective, start, fprime=self.grad, callback=self.debug) + + def optimise_lbfgs(self,start): + print + print "***** LBFGS OPTIMISATION *****" + x,f,d = fmin_l_bfgs_b(self.objective, start, fprime=self.grad, pgtol=1e-09, iprint=0) + return x + +class LRDataException(Exception): + pass + +class LogisticRegressionOptimiser: + """Optimise logistic regression weights""" + def __init__(self,x,y, alpha = 0): + """Training data (x) should be vector of feature vectors, and + corresponding vector of outputs (with values -1,1). + alpha controls the L2-normalisation""" + self.x = x + self.y = y + self.alpha = alpha + if len(x) != len(y): raise LRDataException("Lengths of input and response don't match") + if len(x) == 0: raise LRDataException("Data set is empty") + # Precalculate {y_i*x_ij} for all j + self.xy = x*y[:,None] + + def objective(self,w): + """Calculate the value of the negative log-likelihood for a given weight set""" + l = 0 + for i in range(len(self.x)): + # Each example contributes log(sigma(y_i * x_i . w)) + l -= log(sigmoid(self.y[i] * np.dot(w, self.x[i,:]))) + # regularisation 1/2 * alpha * ||w||^2 + l += 0.5 * self.alpha * np.dot(w,w) + return l + + def grad_j(self,w,j): + """Gradient of the objective in the jth direction for given weight set""" + g = 0 + for i in range(len(self.x)): + # Each example contributes -sigma(-y_i * x_i.w) * y_j x_ij + g -= sigmoid(-self.y[i] * np.dot(w, self.x[i,:])) * self.y[i] * self.x[i,j] + #regularisation + g += self.alpha * w[j] + return g + + def grad(self,w): + """Gradient of objective at given weight set - returns a vector""" + # Calculate the vector -sigma(-y_i * x_i.w) + s = -np.array([sigmoid(-yi * np.dot(xi,w)) for xi,yi in zip(self.x,self.y)]) + # Multiply it by xy + g = np.array([np.dot(xyj,s) for xyj in self.xy.transpose()]) + # Add regularisation + g += self.alpha*w + return g + #g = np.array([self.grad_j(w,j) for j in xrange(len(w))]) + + + def train(self,w0,debug=False): + if debug: + iprint = 0 + else: + iprint = -1 + x,f,d = fmin_l_bfgs_b(self.objective, w0, fprime=self.grad, pgtol=1e-09, iprint=iprint) + if d['warnflag'] != 0: + raise OptimisationException(d['task']) + return x + +class ProTrainer: + """Turns the samples into a logistic regression problem""" + def __init__(self,samples): + self.samples = samples + self.alpha = 1 + self.dims = len(samples[0].hyp1.fv) + + + def train(self, debug=False): + x = np.array([s.hyp1.fv-s.hyp2.fv for s in self.samples]) + #print x + y = np.array([cmp(s.hyp1.score,s.hyp2.score) for s in self.samples]) + #print y + lro = LogisticRegressionOptimiser(x,y,self.alpha) + w0 = np.zeros(self.dims) + w = lro.train(w0,debug) + w = w/np.sum(abs(w)) # L_1 normalise + return w,[] + +class MixtureModelTrainer: + """Trains the phrase mixture weights, as well as the regular feature weights""" + def __init__(self,samples): + self.alpha = 1 + self.interp_floor = 0.001 # minimum value for interpolation weight + #self.prob_floor = 0.00000001 # floor probabilities at this value + #self.weight_bounds = (-10,10) # bounds for other features + # The phrase scores are joined into a 5d array, where the dimensions are: + # sample, hyp1 or hyp2, ttable, phrase-pair, feature + # ie the feature is the last dimension + # Actually phrase_probs is a 2-dim list of 3-dim arrays, since it's ragged + self.phrase_probs = \ + [[sample.hyp1.phrase_scores,sample.hyp2.phrase_scores]\ + for sample in samples] + #[[sample.hyp1.phrase_scores.clip(self.prob_floor),sample.hyp2.phrase_scores.clip(self.prob_floor)]\ + + # Figure out where the weights are + self.phrase_index = list(nbest.get_feature_index("tm")) + self.phrase_index[1] = self.phrase_index[1]-1 # phrase penalty not interpolated + + interp_length = (self.phrase_index[1]-self.phrase_index[0]) * \ + (len(samples[0].hyp1.phrase_scores)-1) + weight_length = len(samples[0].hyp1.fv) + interp_length + self.interp_index = [weight_length - interp_length,weight_length] + #print self.interp_index + self.other_index = [[0,self.phrase_index[0]],[self.phrase_index[1],self.interp_index[0]]] + + # join the feature vector diffs for the other fvs into a 2d array + # features across, samples down + self.fvs = np.array(\ + [np.append(sample.hyp1.fv[self.other_index[0][0]:self.other_index[0][1]], + sample.hyp1.fv[self.other_index[1][0]:self.other_index[1][1]]) - \ + np.append(sample.hyp2.fv[self.other_index[0][0]:self.other_index[0][1]],\ + sample.hyp2.fv[self.other_index[1][0]:self.other_index[1][1]])\ + for sample in samples]) + + self.cached_iw = None + self.cached_interpolated_phrase_probs = None + + self.cached_sw = None + self.cached_y_times_diffs = None + + + + # join the responses (y's) into an array + # If any pairs have equal score, this sets y=0, an invalid response. + # but the sampling should ensure that this doesn't happen + self.y = np.array([cmp(sample.hyp1.score, sample.hyp2.score)\ + for sample in samples]) + + def get_split_weights(self,weights): + """Map containing all the different weight sets: + phrase - phrase feature weights (excluding penalty) + other - other feature weights + interp - interpolation weights: ttable x feature + """ + sw = {} + sw['phrase'] = weights[self.phrase_index[0]:self.phrase_index[1]] + sw['interp'] = weights[self.interp_index[0]:self.interp_index[1]] + sw['interp'] = sw['interp'].T.reshape\ + (( len(sw['interp']) / len(sw['phrase'])), len(sw['phrase'])) + # Add normalisations + sw['interp'] = np.vstack((sw['interp'], 1.0 - np.sum(sw['interp'], axis=0))) + #sw['interp'] = np.append(sw['interp'], 1 - np.sum(sw['interp'])) + sw['other'] = np.append(weights[self.other_index[0][0]:self.other_index[0][1]], + weights[self.other_index[1][0]:self.other_index[1][1]]) + return sw + + def get_interpolated_phrase_probs(self,iw): + # Memoise + if self.cached_iw == None or np.sum(np.abs(iw-self.cached_iw)) != 0: + # iw is ttable x feature. Each element of phrase_probs is ttable x pair x feature + iw_expanded = np.expand_dims(iw,1) + # self.phrase probs is a 2-d list, so use python iteration + interpolated = [ [iw_expanded*p for p in ps] for ps in self.phrase_probs] + self.cached_interpolated_phrase_probs = np.sum(np.array(interpolated), axis = 2) + self.cached_iw = iw + return self.cached_interpolated_phrase_probs + + def get_y_times_diffs(self,sw): + """ Calculate the array y_k* \Delta S_k""" + # Process the phrase scores first. + # - for each phrase, interpolate across the ttables using the current weights + # - sum the log probs across phrase pairs to get a score for each hypothesis + # - take the weighted sum of these scores, to give a phrase feature total + # for each hyp + + # Memoise + if self.cached_sw == None or \ + np.sum(np.abs(self.cached_sw['other'] - sw['other'])) != 0 or \ + np.sum(np.abs(self.cached_sw['phrase'] - sw['phrase'])) != 0 or \ + np.sum(np.abs(self.cached_sw['interp'] - sw['interp'])) != 0: + + # do the interpolation + iw = sw['interp'] + interpolated = self.get_interpolated_phrase_probs(iw) + # Use traditional python as not sure how to vectorise. This goes through + # each hypothesis, logs the probability, applies the feature weights, then sums + self.cached_y_times_diffs = np.zeros(len(interpolated)) + # Take the difference between the hypotheses + for i,sample in enumerate(interpolated): + self.cached_y_times_diffs[i] = \ + np.sum(sw['phrase']* np.log(sample[0])) - \ + np.sum(sw['phrase']* np.log(sample[1])) + #print self.fvs, sw['other'] + #print sw['other'], self.fvs + self.cached_y_times_diffs += np.sum(sw['other'] * self.fvs, axis=1) # add other scores + self.cached_y_times_diffs *= self.y + self.cached_sw = sw + return self.cached_y_times_diffs + + def objective(self,w): + """The value of the objective with the given weight vector. + The objective is the sum of the log of the sigmoid of the differences + in scores between the two hypotheses times y. + """ + diffs = self.get_y_times_diffs(self.get_split_weights(w)) + #print diffs, sigmoid(diffs) + obj = -np.sum(np.log(sigmoid(diffs))) #negative, since minimising + # regularisation + obj += 0.5 * self.alpha * np.dot(w[:self.interp_index[0]], w[:self.interp_index[0]]) + return obj + + # + # The following methods compute the derivatives of the score differences + # with respect to each of the three types of weights. They should all + # return an np.array, with features across, and samples down + # + + def gradient_phrase(self,interp): + """Compute the derivative of the score difference for the 'phrase' weights. + + Args: + interp: The interpolation weights + """ + # Compute the interpolated phrase probs + interpolated = self.get_interpolated_phrase_probs(interp) + + # for each sample, log and sum across phrases, then compute the feature value + # difference for each sample. + # TODO: Better vectorisation + grad_list = [] + for i, sample in enumerate(interpolated): + f_A = np.sum(np.log(sample[0]), axis=0) + f_B = np.sum(np.log(sample[1]), axis=0) + grad_list.append(f_A - f_B) + return np.vstack(grad_list) + + def gradient_interp(self,interp,phrase): + """Compute the derivative of the score difference for the 'interp' weights + + Args: + interp: All the interpolation weights. These will be in a 2-dim np array, + where the dims are ttable x phrase feature. Note that there are k rows, + one for each ttable, so the sum down the columns will be 1. + phrase: The weights of the phrase features + + Returns: + A 2-d array, with samples down and gradients along. Note that in the gradients + (rows) the interpolation weights are flattened out, and have the last ttable + removed. + """ + num_interp_weights = (interp.shape[0]-1) * interp.shape[1] + grad_list = np.empty((len(self.phrase_probs),num_interp_weights)) + expanded_interp = np.expand_dims(interp,1) + def df_by_dlambda(phi): + """Derivative of phrase scores w.r.t. lambdas""" + #print "Interp:", interp, "\nPhi", phi + num = phi[:-1] - phi[-1] + denom = np.sum(expanded_interp*phi, axis=0) + # num is ttable x phrase-pair x feature + # denom is phrase-pair x feature + # divide, then sum across phrase-pairs + #print "num",num,"denom",denom + #print "q",num/denom + quotient = np.sum(num/denom, axis =1) + # quotient is ttable-1 x feature + return quotient + + + for k, sample in enumerate(self.phrase_probs): + # derivative is the weighted sum of df_by_dlambda_A - df_by_dlambda_B + #print "\nq0", df_by_dlambda(sample[0]) + #print "hyp0",np.sum(phrase * (df_by_dlambda(sample[0])), axis=0) + #print "q1", df_by_dlambda(sample[1]) + #print "hyp1",np.sum(phrase * (df_by_dlambda(sample[1])), axis=0),"\n" + #TODO: Check if the sum is required here. With 4 ttables and 4 features + # it gives lhs as (12) and rhs as (4) + grad_list[k] = (phrase * (df_by_dlambda(sample[0]) - df_by_dlambda(sample[1]))).flatten() + #grad_list = np.vstack(grad_list) + return grad_list + + def gradient_other(self): + """Compute the derivative of the score difference for the 'other' weights. + + Features across, samples down. + """ + # This is just the difference in the feature values + return self.fvs + + def gradient(self,w): + sw = self.get_split_weights(w) + sig_y_by_diffs = sigmoid(-self.get_y_times_diffs(sw)) + # These all return 2-d arrays, with samples dowm and features across. + # NB: Both gradient_phrase and gradient_interp iterate through the samples, + # so this is probably inefficient + phrase_g = self.gradient_phrase(sw['interp']) + interp_g = self.gradient_interp(sw['interp'], sw['phrase']) + other_g = self.gradient_other() + + # For each feature, we get the gradient by multiplying by \sigma (-y*\Delta S), + # multiplying by y, and summing across all samples + # Take negatives since we're minimising + phrase_g = -np.sum(np.transpose(phrase_g) * sig_y_by_diffs * self.y, axis=1) + interp_g = -np.sum(np.transpose(interp_g) * sig_y_by_diffs * self.y, axis=1) + other_g = -np.sum(np.transpose(other_g) * sig_y_by_diffs * self.y, axis=1) + + # regularisation + phrase_g += self.alpha * sw['phrase'] + other_g += self.alpha * sw['other'] + + # Splice the gradients together + grad = np.array([0.0]* len(w)) + grad[-len(interp_g):] = interp_g + grad[self.phrase_index[0]:self.phrase_index[1]] = phrase_g + grad[self.other_index[0][0]:self.other_index[0][1]] = \ + other_g[:self.other_index[0][1] - self.other_index[0][0]] + grad[self.other_index[1][0]:self.other_index[1][1]] = \ + other_g[self.other_index[0][1] - self.other_index[0][0]:] + return grad + + + def train(self,debug=False): + """Train the mixture model.""" + if debug: + iprint = 0 + else: + iprint = -1 + # Initialise weights to zero, except interpolation + num_phrase_features = self.phrase_index[1] - self.phrase_index[0] + num_models = ((self.interp_index[1] - self.interp_index[0])/num_phrase_features)+1 + w0 = [0.0] * self.interp_index[0] + w0 += [1.0/num_models] * (self.interp_index[1]-self.interp_index[0]) + bounds = [(None,None)] * len(w0) + bounds[self.interp_index[0]:self.interp_index[1]] = \ + [(self.interp_floor,1)] * (self.interp_index[1] - self.interp_index[0]) + w0 = np.array(w0) + x,f,d = fmin_l_bfgs_b(self.objective, w0, fprime=self.gradient, bounds=bounds, pgtol=1e-09, iprint=iprint) + if d['warnflag'] != 0: + raise OptimisationException(d['task']) + weights = x[:self.interp_index[0]] + mix_weights = x[self.interp_index[0]:] + mix_weights = mix_weights.reshape((num_models-1,num_phrase_features)) + mix_weights = np.vstack((mix_weights, 1-np.sum(mix_weights,axis=0))) + return weights,mix_weights + + +# +# Test logistic regression using pro data +# +def main(): + fh = open("data/esen.wmt12.pro") + x = [] + y = [] + d = 14 + for line in fh: + line = line[:-1] + fields = line.split() + if fields[0] == "1": + y.append(1) + else: + y.append(-1) + x_i = [0]*d + for i in xrange(1,len(fields),2): + j = int(fields[i][1:]) + x_ij = float(fields[i+1]) + x_i[j] = x_ij + x.append(x_i) + lro = LogisticRegressionOptimiser(np.array(x), np.array(y), 0.1) + print lro.train(np.zeros(d), debug=True) + + +if __name__ == "__main__": + main() diff --git a/mosesdecoder/contrib/relent-filter/scripts/calcEmpiricalDistribution.pl b/mosesdecoder/contrib/relent-filter/scripts/calcEmpiricalDistribution.pl new file mode 100644 index 0000000000000000000000000000000000000000..462ec533938206005ac0c7a1f87d9edb376e5002 --- /dev/null +++ b/mosesdecoder/contrib/relent-filter/scripts/calcEmpiricalDistribution.pl @@ -0,0 +1,53 @@ +#!/usr/bin/perl -w + +# read arguments +my $countFile = $ARGV[0]; + +my $ZCAT = "gzip -cd"; +my $BZCAT = "bzcat"; + +&process_count_file($countFile); + +sub process_count_file { + $file = $_[0]; + open(COUNT_READER, &open_compressed($file)) or die "ERROR: Can't read $file"; + + print STDERR "reading file to calculate normalizer"; + $normalizer=0; + while() { + my $line = $_; + chomp($line); + my @line_array = split(/\s+/, $line); + my $count = $line_array[0]; + $normalizer+=$count; + } + + close(COUNT_READER); + + print STDERR "reading file again to print the counts"; + open(COUNT_READER, &open_compressed($file)) or die "ERROR: Can't read $file"; + + while() { + my $line = $_; + chomp($line); + my @line_array = split(/\s+/, $line); + my $score = $line_array[0]/$normalizer; + print $score."\n"; + } + + close(COUNT_READER); +} + +sub open_compressed { + my ($file) = @_; + print STDERR "FILE: $file\n"; + + # add extensions, if necessary + $file = $file.".bz2" if ! -e $file && -e $file.".bz2"; + $file = $file.".gz" if ! -e $file && -e $file.".gz"; + + # pipe zipped, if necessary + return "$BZCAT $file|" if $file =~ /\.bz2$/; + return "$ZCAT $file|" if $file =~ /\.gz$/; + return $file; +} diff --git a/mosesdecoder/contrib/relent-filter/scripts/calcPruningScores.pl b/mosesdecoder/contrib/relent-filter/scripts/calcPruningScores.pl new file mode 100644 index 0000000000000000000000000000000000000000..cbfabac55b002c0b019dda61ebeba94a9b5c7c1c --- /dev/null +++ b/mosesdecoder/contrib/relent-filter/scripts/calcPruningScores.pl @@ -0,0 +1,351 @@ +#!/usr/bin/perl -w +use Getopt::Long; +use File::Basename; +use POSIX; + +# read arguments +my $line_start = 0; +my $line_end = LONG_MAX; +my $tmp_dir = ""; +my $dec_size = LONG_MAX; +$_HELP = 1 if (@ARGV < 1 or !GetOptions ("moses_ini=s" => \$moses_ini, #moses conf file +"start:i" => \$line_start, #fisrt phrase to process +"end:i" => \$line_end, #last sentence to process (not including) +"training_s=s" => \$training_s, #source training file +"training_t=s" => \$training_t, #target training file +"prune_bin=s" => \$prune_bin, #binary files in the pruning toolkit +"prune_scripts=s" => \$prune_scripts, #scripts in the pruning toolkit +"sig_bin=s" => \$sig_bin, #binary files to calculate significance +"moses_scripts=s" => \$moses_scripts, #dir with the moses scripts +"tmp_dir:s" => \$tmp_dir, #dir with the moses scripts +"dec_size:i" => \$dec_size, #dir with the moses scripts +"workdir=s" => \$workdir)); #directory to put all the output files + +# help message if arguments are not correct +if ($_HELP) { + print " +Usage: perl calcPruningScores.pl [PARAMS] +Function: Calculates relative entropy for each phrase pair in a translation model. +Authors: Wang Ling ( lingwang at cs dot cmu dot edu ) +PARAMS: + -moses_ini : moses configuration file with the model to prune (phrase table, reordering table, weights etc...) + -training_s : source training file, please run salm first + -training_t : target training file, please run salm first + -prune_bin : path to the binaries for pruning (probably /bin) + -prune_scripts : path to the scripts for pruning (probably the directory where this script is) + -sig_bin : path to the binary for significance testing included in this toolkit + -moses_scripts : path to the moses training scripts (where filter-model-given-input.pl is) + -workdir : directory to produce the output + -tmp_dir : directory to store temporary files (improve performance if stored in a local disk), omit to store in workdir + -dec_size : number of phrase pairs to be decoded at a time, omit to decode all selected phrase pairs at once + -start and -end : starting and ending phrase pairs to process, to be used if you want to launch multiple processes in parallel for different parts of the phrase table. If specified the process will process the phrase pairs from to + +For any questions contact lingwang at cs dot cmu dot edu +"; + exit(1); +} + +# setting up working dirs +my $TMP_DIR = $tmp_dir; +if ($tmp_dir eq ""){ + $TMP_DIR = "$workdir/tmp"; +} +my $SCORE_DIR = "$workdir/scores"; +my $FILTER_DIR = "$TMP_DIR/filter"; + +# files for divergence module +my $SOURCE_FILE = "$TMP_DIR/source.txt"; +my $CONSTRAINT_FILE = "$TMP_DIR/constraint.txt"; +my $DIVERGENCE_FILE = "$SCORE_DIR/divergence.txt"; + +# files for significance module +my $SIG_TABLE_FILE = "$TMP_DIR/source_target.txt"; +my $SIG_MOD_OUTPUT = "$TMP_DIR/sig_mod.out"; +my $SIG_FILE = "$SCORE_DIR/significance.txt"; +my $COUNT_FILE = "$SCORE_DIR/count.txt"; +my $EMP_DIST_FILE= "$SCORE_DIR/empirical.txt"; +my $REL_ENT_FILE= "$SCORE_DIR/rel_ent.txt"; + +# setting up executables +my $ZCAT = "gzip -cd"; +my $BZCAT = "bzcat"; +my $CP = "cp"; +my $SED = "sed"; +my $RM = "rm"; +my $SORT_EXEC = "sort"; +my $PRUNE_EXEC = "$prune_bin/calcDivergence"; +my $SIG_EXEC = "$sig_bin/filter-pt"; +my $FILTER_EXEC = "perl $moses_scripts/filter-model-given-input.pl"; +my $CALC_EMP_EXEC ="perl $prune_scripts/calcEmpiricalDistribution.pl"; +my $INT_TABLE_EXEC = "perl $prune_scripts/interpolateScores.pl"; + +# moses ini variables +my ($TRANSLATION_TABLE_FILE, $REORDERING_TABLE_FILE); + +# phrase table variables +my ($N_PHRASES, $N_PHRASES_TO_PROCESS); + +# main functions +&prepare(); +&calc_sig_and_counts(); +&calc_div(); +&clear_up(); + +# (1) preparing data +sub prepare { + print STDERR "(1) preparing data @ ".`date`; + safesystem("mkdir -p $workdir") or die("ERROR: could not create work dir $workdir"); + safesystem("mkdir -p $TMP_DIR") or die("ERROR: could not create work dir $TMP_DIR"); + safesystem("mkdir -p $SCORE_DIR") or die("ERROR: could not create work dir $SCORE_DIR"); + &get_moses_ini_params(); + ©_tables_to_tmp_dir(); + &write_data_files(); + + $N_PHRASES = &get_number_of_phrases(); + $line_end = ($line_end > $N_PHRASES) ? $N_PHRASES : $line_end; + $N_PHRASES_TO_PROCESS = $line_end - $line_start; +} + +sub write_data_files { + open(SOURCE_WRITER,">".$SOURCE_FILE) or die "ERROR: Can't write $SOURCE_FILE"; + open(CONSTRAINT_WRITER,">".$CONSTRAINT_FILE) or die "ERROR: Can't write $CONSTRAINT_FILE"; + open(TABLE_WRITER,">".$SIG_TABLE_FILE) or die "ERROR: Can't write $SIG_TABLE_FILE"; + open(TTABLE_READER, &open_compressed($TRANSLATION_TABLE_FILE)) or die "ERROR: Can't read $TRANSLATION_TABLE_FILE"; + + $line_number = 0; + while($line_number < $line_start && !eof(TTABLE_READER)){ + ; + $line_number++; + } + while($line_number < $line_end && !eof(TTABLE_READER)) { + my $line = ; + chomp($line); + my @line_array = split(/\s+\|\|\|\s+/, $line); + my $source = $line_array[0]; + my $target = $line_array[1]; + my $scores = $line_array[2]; + print TABLE_WRITER $source." ||| ".$target." ||| ".$scores."\n"; + print SOURCE_WRITER $source."\n"; + print CONSTRAINT_WRITER $target."\n"; + $line_number++; + } + + close(SOURCE_WRITER); + close(CONSTRAINT_WRITER); + close(TABLE_WRITER); + close(TTABLE_READER); +} + +sub copy_tables_to_tmp_dir { + $tmp_t_table = "$TMP_DIR/".basename($TRANSLATION_TABLE_FILE); + $tmp_r_table = "$TMP_DIR/".basename($REORDERING_TABLE_FILE); + $tmp_moses_ini = "$TMP_DIR/moses.ini"; + $cp_t_cmd = "$CP $TRANSLATION_TABLE_FILE $TMP_DIR"; + $cp_r_cmd = "$CP $REORDERING_TABLE_FILE $TMP_DIR"; + safesystem("$cp_t_cmd") or die("ERROR: could not run:\n $cp_t_cmd"); + safesystem("$cp_r_cmd") or die("ERROR: could not run:\n $cp_r_cmd"); + + $sed_cmd = "$SED s#$TRANSLATION_TABLE_FILE#$tmp_t_table#g $moses_ini | $SED s#$REORDERING_TABLE_FILE#$tmp_r_table#g > $tmp_moses_ini"; + safesystem("$sed_cmd") or die("ERROR: could not run:\n $sed_cmd"); + + $TRANSLATION_TABLE_FILE = $tmp_t_table; + $REORDERING_TABLE_FILE = $tmp_r_table; + $moses_ini = $tmp_moses_ini; +} + +# (2) calculating sig and counts +sub calc_sig_and_counts { + print STDERR "(2) calculating counts and significance".`date`; + print STDERR "(2.1) running significance module".`date`; + &run_significance_module(); + print STDERR "(2.2) writing counts and significance tables".`date`; + &write_counts_and_significance_table(); + print STDERR "(2.3) calculating empirical distribution".`date`; +} + +sub write_counts_and_significance_table { + open(COUNT_WRITER,">".$COUNT_FILE) or die "ERROR: Can't write $COUNT_FILE"; + open(SIG_WRITER,">".$SIG_FILE) or die "ERROR: Can't write $SIG_FILE"; + open(SIG_MOD_READER, &open_compressed($SIG_MOD_OUTPUT)) or die "ERROR: Can't read $SIG_MOD_OUTPUT"; + + while() { + my($line) = $_; + chomp($line); + my @line_array = split(/\s+\|\|\|\s+/, $line); + my $count = $line_array[0]; + my $sig = $line_array[1]; + print COUNT_WRITER $count."\n"; + print SIG_WRITER $sig."\n"; + } + + close(SIG_MOD_READER); + close(COUNT_WRITER); + close(SIG_WRITER); +} + +sub run_significance_module { + my $sig_cmd = "cat $SIG_TABLE_FILE | $SIG_EXEC -e $training_t -f $training_s -l -10000 -p -c > $SIG_MOD_OUTPUT"; + safesystem("$sig_cmd") or die("ERROR: could not run:\n $sig_cmd"); +} + +# (3) calculating divergence +sub calc_div { + print STDERR "(3) calculating relative entropy".`date`; + print STDERR "(3.1) calculating empirical distribution".`date`; + &calculate_empirical_distribution(); + print STDERR "(3.2) calculating divergence (this might take a while)".`date`; + if($N_PHRASES_TO_PROCESS > $dec_size) { + &calculate_divergence_shared("$FILTER_DIR"); + } + else{ + &calculate_divergence($moses_ini); + } + print STDERR "(3.3) calculating relative entropy from empirical and divergence distributions".`date`; + &calculate_relative_entropy(); +} + +sub calculate_empirical_distribution { + my $emp_cmd = "$CALC_EMP_EXEC $COUNT_FILE > $EMP_DIST_FILE"; + safesystem("$emp_cmd") or die("ERROR: could not run:\n $emp_cmd"); +} + +sub get_fragmented_file_name { + my ($name, $frag, $interval) = @_; + return "$name-$frag-".($frag+$interval); +} + +sub calculate_divergence { + my $moses_ini_file = $_[0]; + print STDERR "force decoding phrase pairs\n"; + my $prune_cmd = "cat $SOURCE_FILE | $PRUNE_EXEC -f $moses_ini_file -constraint $CONSTRAINT_FILE -early-discarding-threshold 0 -s 100000 -ttable-limit 0 > $DIVERGENCE_FILE 2> /dev/null"; + safesystem("$prune_cmd") or die("ERROR: could not run:\n $prune_cmd"); +} + +sub calculate_divergence_shared { + my $filter_dir = $_[0]; + + &split_file_into_chunks($SOURCE_FILE, $dec_size, $N_PHRASES_TO_PROCESS); + &split_file_into_chunks($CONSTRAINT_FILE, $dec_size, $N_PHRASES_TO_PROCESS); + + for(my $i = 0; $i < $N_PHRASES_TO_PROCESS; $i = $i + $dec_size) { + my $filter_cmd = "$FILTER_EXEC ".&get_fragmented_file_name($FILTER_DIR, $i, $dec_size)." $moses_ini ".&get_fragmented_file_name($SOURCE_FILE, $i, $dec_size); + safesystem("$filter_cmd") or die("ERROR: could not run:\n $filter_cmd"); + + my $moses_ini_file = &get_fragmented_file_name($filter_dir, $i, $dec_size)."/moses.ini"; + my $source_file = &get_fragmented_file_name($SOURCE_FILE, $i, $dec_size); + my $constraint_file = &get_fragmented_file_name($CONSTRAINT_FILE, $i, $dec_size); + my $prune_cmd; + print STDERR "force decoding phrase pairs $i to ".($i + $dec_size)."\n"; + if($i == 0){ + $prune_cmd = "cat $source_file | $PRUNE_EXEC -f $moses_ini_file -constraint $constraint_file -early-discarding-threshold 0 -s 100000 -ttable-limit 0 > $DIVERGENCE_FILE 2> /dev/null"; + } + else{ + $prune_cmd = "cat $source_file | $PRUNE_EXEC -f $moses_ini_file -constraint $constraint_file -early-discarding-threshold 0 -s 100000 -ttable-limit 0 >> $DIVERGENCE_FILE 2> /dev/null"; + } + safesystem("$prune_cmd") or die("ERROR: could not run:\n $prune_cmd"); + + my $rm_cmd = "$RM -r ".&get_fragmented_file_name($FILTER_DIR, $i, $dec_size); + safesystem("$rm_cmd") or die("ERROR: could not run:\n $rm_cmd"); + + } +} + +sub calculate_relative_entropy { + my $int_cmd = "$INT_TABLE_EXEC -files \"$EMP_DIST_FILE $DIVERGENCE_FILE\" -weights \"1 1\" -operation \"*\" > $REL_ENT_FILE"; + safesystem("$int_cmd") or die("ERROR: could not run:\n $int_cmd"); + +} + +# (4) clear up stuff that is not needed +sub clear_up { + print STDERR "(4) removing tmp dir".`date`; + $rm_cmd = "$RM -r $TMP_DIR"; + safesystem("$rm_cmd") or die("ERROR: could not run:\n $rm_cmd"); +} + +# utility functions + +sub safesystem { + print STDERR "Executing: @_\n"; + system(@_); + if ($? == -1) { + print STDERR "ERROR: Failed to execute: @_\n $!\n"; + exit(1); + } + elsif ($? & 127) { + printf STDERR "ERROR: Execution of: @_\n died with signal %d, %s coredump\n", + ($? & 127), ($? & 128) ? 'with' : 'without'; + exit(1); + } + else { + my $exitcode = $? >> 8; + print STDERR "Exit code: $exitcode\n" if $exitcode; + return ! $exitcode; + } +} + +sub open_compressed { + my ($file) = @_; + print STDERR "FILE: $file\n"; + + # add extensions, if necessary + $file = $file.".bz2" if ! -e $file && -e $file.".bz2"; + $file = $file.".gz" if ! -e $file && -e $file.".gz"; + + # pipe zipped, if necessary + return "$BZCAT $file|" if $file =~ /\.bz2$/; + return "$ZCAT $file|" if $file =~ /\.gz$/; + return $file; +} + +sub get_moses_ini_params { + + open(MOSES_READER, $moses_ini); + while() { + my($line) = $_; + chomp($line); + + if($line eq "[ttable-file]"){ + $tableLine = ; + chomp($tableLine); + ($_,$_,$_,$_,$TRANSLATION_TABLE_FILE) = split(" ",$tableLine); # put the other parameters there if needed + } + if($line eq "[distortion-file]"){ + $tableLine = ; + chomp($tableLine); + ($_,$_,$_,$REORDERING_TABLE_FILE) = split(" ",$tableLine); # put the other parameters there if needed + } + } + close(MOSES_READER); +} + +sub get_number_of_phrases { + my $ret = 0; + open(TABLE_READER, &open_compressed($TRANSLATION_TABLE_FILE)) or die "ERROR: Can't read $TRANSLATION_TABLE_FILE"; + + while() { + $ret++; + } + + close (TABLE_READER); + return $ret; +} + +sub split_file_into_chunks { + my ($file_to_split, $chunk_size, $number_of_phrases_to_process) = @_; + open(SOURCE_READER, &open_compressed($file_to_split)) or die "ERROR: Can't read $file_to_split"; + my $FRAG_SOURCE_WRITER; + for(my $i = 0; $i < $number_of_phrases_to_process && !eof(SOURCE_READER); $i++) { + if(($i % $chunk_size) == 0){ # open fragmented file to write + my $frag_file = &get_fragmented_file_name($file_to_split, $i, $chunk_size); + open(FRAG_SOURCE_WRITER, ">".$frag_file) or die "ERROR: Can't write $frag_file"; + } + my $line = ; + print FRAG_SOURCE_WRITER $line; + if((%i % $chunk_size) == $chunk_size - 1 || (%i % $chunk_size) == $number_of_phrases_to_process - 1){ # close fragmented file before opening a new one + close(FRAG_SOURCE_WRITER); + } + } +} + + diff --git a/mosesdecoder/contrib/relent-filter/scripts/interpolateScores.pl b/mosesdecoder/contrib/relent-filter/scripts/interpolateScores.pl new file mode 100644 index 0000000000000000000000000000000000000000..b204e951aeb1ac69aac1493674963baa47aaa57e --- /dev/null +++ b/mosesdecoder/contrib/relent-filter/scripts/interpolateScores.pl @@ -0,0 +1,94 @@ +#!/usr/bin/perl -w +use Getopt::Long; +use File::Basename; +use POSIX; + +$operation="+"; + +# read arguments +$_HELP = 1 if (@ARGV < 1 or !GetOptions ("files=s" => \$files, #moses conf file +"weights=s" => \$weights, +"operation=s" => \$operation)); #directory to put all the output files + + +# help message if arguments are not correct +if ($_HELP) { + print "Relative Entropy Pruning +Usage: perl interpolateScores.pl [PARAMS] +Function: interpolates any number of score files interlated by their weights +Authors: Wang Ling ( lingwang at cs dot cmu dot edu ) +PARAMS: + -files=s : table files to interpolate separated by a space (Ex \"file1 file2 file3\") + -weights : interpolation weights separated by a space (Ex \"0.3 0.3 0.4\") + -operation : +,* or min depending on the operation to perform to combine scores +For any questions contact lingwang at cs dot cmu dot edu +"; + exit(1); +} + +@FILES = split(/\s+/, $files); +@WEIGHTS = split(/\s+/, $weights); + +my $ZCAT = "gzip -cd"; +my $BZCAT = "bzcat"; + +&interpolate(); + +sub interpolate { + my @READERS; + for($i = 0; $i < @FILES; $i++){ + local *FILE; + open(FILE, &open_compressed($FILES[$i])) or die "ERROR: Can't read $FILES[$i]"; + push(@READERS, *FILE); + } + $FIRST = $READERS[0]; + while(!eof($FIRST)) { + if($operation eq "+"){ + my $score = 0; + for($i = 0; $i < @FILES; $i++){ + my $READER = $READERS[$i]; + my $line = <$READER>; + chomp($line); + $score += $line*$WEIGHTS[$i]; + } + print "$score\n"; + } + if($operation eq "*"){ + my $score = 1; + for($i = 0; $i < @FILES; $i++){ + my $READER = $READERS[$i]; + my $line = <$READER>; + chomp($line); + $score *= $line ** $WEIGHTS[$i]; + } + print "$score\n" + } + if($operation eq "min"){ + my $score = 99999; + for($i = 0; $i < @FILES; $i++){ + my $READER = $READERS[$i]; + my $line = <$READER>; + chomp($line); + if ($score > $line*$WEIGHTS[$i]){ + $score = $line*$WEIGHTS[$i]; + } + } + print "$score\n" + + } + } +} + +sub open_compressed { + my ($file) = @_; + print STDERR "FILE: $file\n"; + + # add extensions, if necessary + $file = $file.".bz2" if ! -e $file && -e $file.".bz2"; + $file = $file.".gz" if ! -e $file && -e $file.".gz"; + + # pipe zipped, if necessary + return "$BZCAT $file|" if $file =~ /\.bz2$/; + return "$ZCAT $file|" if $file =~ /\.gz$/; + return $file; +} diff --git a/mosesdecoder/contrib/relent-filter/scripts/prunePT.pl b/mosesdecoder/contrib/relent-filter/scripts/prunePT.pl new file mode 100644 index 0000000000000000000000000000000000000000..b443cc14e0d8b80a3034affdffa6c9e094ece1f4 --- /dev/null +++ b/mosesdecoder/contrib/relent-filter/scripts/prunePT.pl @@ -0,0 +1,114 @@ +#!/usr/bin/perl -w + +# read arguments +my $tmp_dir = ""; +my $percentage = -1; +my $threshold = -1; +use Getopt::Long; +$_HELP = 1 if (@ARGV < 1 or !GetOptions ("table=s" => \$table, #table to filter +"scores=s" => \$scores_file, #scores of each phrase pair, should have same size as the table to filter +"percentage=f" => \$percentage, # percentage of phrase table to remain +"threshold=f" => \$threshold)); # threshold (score < threshold equals prune entry) + +# help message if arguments are not correct +if ($_HELP) { + print "Relative Entropy Pruning +Usage: perl prunePT.pl [PARAMS] +Function: prunes a phrase table given a score file +Authors: Wang Ling ( lingwang at cs dot cmu dot edu ) +PARAMS: + -table : table to prune + -percentage : percentage of phrase table to remain (if the scores do not allow the exact percentage if multiple entries have the same threshold, the script chooses to retain more than the given percentage) + -threshold : threshold to prune (score < threshold equals prune entry), do not use this if percentage is specified +For any questions contact lingwang at cs dot cmu dot edu +"; + exit(1); +} + + +my $THRESHOLD = $threshold; +if ($percentage != -1){ + $THRESHOLD = &get_threshold_by_percentage($percentage); +} + +my $ZCAT = "gzip -cd"; +my $BZCAT = "bzcat"; + +&prune_by_threshold($THRESHOLD); + +sub prune_by_threshold { + my $th = $_[0]; + print STDERR "pruning using threshold $th \n"; + open (SCORE_READER, &open_compressed($scores_file)); + open (TABLE_READER, &open_compressed($table)); + $number_of_phrases=0; + $number_of_unpruned_phrases=0; + while(!eof(SCORE_READER) && !eof(TABLE_READER)){ + $score_line = ; + $table_line = ; + chomp($score_line); + if($score_line >= $th){ + print $table_line; + $number_of_unpruned_phrases++; + } + $number_of_phrases++; + } + print STDERR "pruned ".($number_of_phrases - $number_of_unpruned_phrases)." phrase pairs out of $number_of_phrases\n"; +} + +sub get_threshold_by_percentage { + my $percentage = $_[0]; + $ret = 0; + + $number_of_phrases = &get_number_of_phrases(); + $stop_phrase = ($percentage * $number_of_phrases) / 100; + $phrase_number = 0; + + + open (SCORE_READER, &open_compressed($scores_file)); + while() { + my $line = $_; + + } + close (SCORE_READER); + + open (SCORE_READER, "cat $scores_file | LC_ALL=c sort -g |"); + while() { + my $line = $_; + if($phrase_number >= $stop_phrase){ + chomp($line); + $ret = $line; + last; + } + $phrase_number++; + } + + close (SCORE_READER); + return $ret; +} + +sub get_number_of_phrases { + $ret = 0; + open (SCORE_READER, $scores_file); + + while() { + $ret++; + } + + close (SCORE_READER); + return $ret; +} + +sub open_compressed { + my ($file) = @_; + print STDERR "FILE: $file\n"; + + # add extensions, if necessary + $file = $file.".bz2" if ! -e $file && -e $file.".bz2"; + $file = $file.".gz" if ! -e $file && -e $file.".gz"; + + # pipe zipped, if necessary + return "$BZCAT $file|" if $file =~ /\.bz2$/; + return "$ZCAT $file|" if $file =~ /\.gz$/; + return $file; +} diff --git a/mosesdecoder/contrib/relent-filter/sigtest-filter/Makefile b/mosesdecoder/contrib/relent-filter/sigtest-filter/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..71de9c45fa6dda81f278bc4106d5f5fd1cefa94f --- /dev/null +++ b/mosesdecoder/contrib/relent-filter/sigtest-filter/Makefile @@ -0,0 +1,10 @@ +SALMDIR=/Users/hieuhoang/workspace/salm +FLAVOR?=o64 +INC=-I$(SALMDIR)/Src/Shared -I$(SALMDIR)/Src/SuffixArrayApplications -I$(SALMDIR)/Src/SuffixArrayApplications/SuffixArraySearch +OBJS=$(SALMDIR)/Distribution/Linux/Objs/Search/_SuffixArrayApplicationBase.$(FLAVOR) $(SALMDIR)/Distribution/Linux/Objs/Search/_SuffixArraySearchApplicationBase.$(FLAVOR) $(SALMDIR)/Distribution/Linux/Objs/Shared/_String.$(FLAVOR) $(SALMDIR)/Distribution/Linux/Objs/Shared/_IDVocabulary.$(FLAVOR) + +all: filter-pt + +filter-pt: filter-pt.cpp + ./check-install $(SALMDIR) + $(CXX) -O6 $(INC) $(OBJS) -o filter-pt filter-pt.cpp diff --git a/mosesdecoder/contrib/relent-filter/sigtest-filter/WIN32_functions.h b/mosesdecoder/contrib/relent-filter/sigtest-filter/WIN32_functions.h new file mode 100644 index 0000000000000000000000000000000000000000..ad644018bd55c2d057bc30a661da6b5eda5d7e75 --- /dev/null +++ b/mosesdecoder/contrib/relent-filter/sigtest-filter/WIN32_functions.h @@ -0,0 +1,24 @@ +// XGetopt.h Version 1.2 +// +// Author: Hans Dietrich +// hdietrich2@hotmail.com +// +// This software is released into the public domain. +// You are free to use it in any way you like. +// +// This software is provided "as is" with no expressed +// or implied warranty. I accept no liability for any +// damage or loss of business that this software may cause. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef XGETOPT_H +#define XGETOPT_H + +extern int optind, opterr; +extern char *optarg; + +int getopt(int argc, char *argv[], char *optstring); +double lgamma(int x); + +#endif //XGETOPT_H diff --git a/mosesdecoder/contrib/relent-filter/sigtest-filter/filter-pt.cpp b/mosesdecoder/contrib/relent-filter/sigtest-filter/filter-pt.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e2408900dc2d820bd77b9f68b73e60be7282e7bc --- /dev/null +++ b/mosesdecoder/contrib/relent-filter/sigtest-filter/filter-pt.cpp @@ -0,0 +1,377 @@ + +#include +#include +#include +#include +#include + +#include "_SuffixArraySearchApplicationBase.h" + +#include +#include +#include + +#ifdef WIN32 +#include "WIN32_functions.h" +#else +#include +#endif + +typedef std::set SentIdSet; +typedef std::map PhraseSetMap; + +#undef min + +// constants +const size_t MINIMUM_SIZE_TO_KEEP = 10000; // reduce this to improve memory usage, +// increase for speed +const std::string SEPARATOR = " ||| "; + +const double ALPHA_PLUS_EPS = -1000.0; // dummy value +const double ALPHA_MINUS_EPS = -2000.0; // dummy value + +// configuration params +int pfe_filter_limit = 0; // 0 = don't filter anything based on P(f|e) +bool print_cooc_counts = false; // add cooc counts to phrase table? +bool print_neglog_significance = false; // add -log(p) to phrase table? +double sig_filter_limit = 0; // keep phrase pairs with -log(sig) > sig_filter_limit +// higher = filter-more +bool pef_filter_only = false; // only filter based on pef + +// globals +PhraseSetMap esets; +double p_111 = 0.0; // alpha +size_t nremoved_sigfilter = 0; +size_t nremoved_pfefilter = 0; + +C_SuffixArraySearchApplicationBase e_sa; +C_SuffixArraySearchApplicationBase f_sa; +int num_lines; + +void usage() +{ + std::cerr << "\nFilter phrase table using significance testing as described\n" + << "in H. Johnson, et al. (2007) Improving Translation Quality\n" + << "by Discarding Most of the Phrasetable. EMNLP 2007.\n" + << "\nUsage:\n" + << "\n filter-pt -e english.suf-arr -f french.suf-arr\n" + << " [-c] [-p] [-l threshold] [-n num] < PHRASE-TABLE > FILTERED-PHRASE-TABLE\n\n" + << " [-l threshold] >0.0, a+e, or a-e: keep values that have a -log significance > this\n" + << " [-n num ] 0, 1...: 0=no filtering, >0 sort by P(e|f) and keep the top num elements\n" + << " [-c ] add the cooccurence counts to the phrase table\n" + << " [-p ] add -log(significance) to the phrasetable\n\n"; + exit(1); +} + +struct PTEntry { + PTEntry(const std::string& str, int index); + std::string f_phrase; + std::string e_phrase; + std::string extra; + std::string scores; + float pfe; + int cf; + int ce; + int cfe; + float nlog_pte; + void set_cooc_stats(int _cef, int _cf, int _ce, float nlp) { + cfe = _cef; + cf = _cf; + ce = _ce; + nlog_pte = nlp; + } + +}; + +PTEntry::PTEntry(const std::string& str, int index) : + cf(0), ce(0), cfe(0), nlog_pte(0.0) +{ + size_t pos = 0; + std::string::size_type nextPos = str.find(SEPARATOR, pos); + this->f_phrase = str.substr(pos,nextPos); + + pos = nextPos + SEPARATOR.size(); + nextPos = str.find(SEPARATOR, pos); + this->e_phrase = str.substr(pos,nextPos-pos); + + pos = nextPos + SEPARATOR.size(); + nextPos = str.find(SEPARATOR, pos); + this->scores = str.substr(pos,nextPos-pos); + + pos = nextPos + SEPARATOR.size(); + this->extra = str.substr(pos); + + int c = 0; + std::string::iterator i=scores.begin(); + if (index > 0) { + for (; i != scores.end(); ++i) { + if ((*i) == ' ') { + c++; + if (c == index) break; + } + } + } + if (i != scores.end()) { + ++i; + } + char f[24]; + char *fp=f; + while (i != scores.end() && *i != ' ') { + *fp++=*i++; + } + *fp++=0; + + this->pfe = atof(f); + + // std::cerr << "L: " << f_phrase << " ::: " << e_phrase << " ::: " << scores << " ::: " << pfe << std::endl; + // std::cerr << "X: " << extra << "\n"; +} + +struct PfeComparer { + bool operator()(const PTEntry* a, const PTEntry* b) const { + return a->pfe > b->pfe; + } +}; + +struct NlogSigThresholder { + NlogSigThresholder(float threshold) : t(threshold) {} + float t; + bool operator()(const PTEntry* a) const { + if (a->nlog_pte < t) { + delete a; + return true; + } else return false; + } +}; + +std::ostream& operator << (std::ostream& os, const PTEntry& pp) +{ + //os << pp.f_phrase << " ||| " << pp.e_phrase; + //os << " ||| " << pp.scores; + //if (pp.extra.size()>0) os << " ||| " << pp.extra; + if (print_cooc_counts) os << pp.cfe << " " << pp.cf << " " << pp.ce; + if (print_neglog_significance) os << " ||| " << pp.nlog_pte; + return os; +} + +void print(int a, int b, int c, int d, float p) +{ + std::cerr << a << "\t" << b << "\t P=" << p << "\n" + << c << "\t" << d << "\t xf=" << (double)(b)*(double)(c)/(double)(a+1)/(double)(d+1) << "\n\n"; +} + +// 2x2 (one-sided) Fisher's exact test +// see B. Moore. (2004) On Log Likelihood and the Significance of Rare Events +double fisher_exact(int cfe, int ce, int cf) +{ + assert(cfe <= ce); + assert(cfe <= cf); + + int a = cfe; + int b = (cf - cfe); + int c = (ce - cfe); + int d = (num_lines - ce - cf + cfe); + int n = a + b + c + d; + + double cp = exp(lgamma(1+a+c) + lgamma(1+b+d) + lgamma(1+a+b) + lgamma(1+c+d) - lgamma(1+n) - lgamma(1+a) - lgamma(1+b) - lgamma(1+c) - lgamma(1+d)); + double total_p = 0.0; + int tc = std::min(b,c); + for (int i=0; i<=tc; i++) { + total_p += cp; +// double lg = lgamma(1+a+c) + lgamma(1+b+d) + lgamma(1+a+b) + lgamma(1+c+d) - lgamma(1+n) - lgamma(1+a) - lgamma(1+b) - lgamma(1+c) - lgamma(1+d); double cp = exp(lg); +// print(a,b,c,d,cp); + double coef = (double)(b)*(double)(c)/(double)(a+1)/(double)(d+1); + cp *= coef; + ++a; + --c; + ++d; + --b; + } + return total_p; +} + +// input: unordered list of translation options for a single source phrase +void compute_cooc_stats_and_filter(std::vector& options) +{ + if (pfe_filter_limit>0 && options.size() > pfe_filter_limit) { + nremoved_pfefilter += (options.size() - pfe_filter_limit); + std::nth_element(options.begin(), options.begin()+pfe_filter_limit, options.end(), PfeComparer()); + for (std::vector::iterator i=options.begin()+pfe_filter_limit; i != options.end(); ++i) + delete *i; + options.erase(options.begin()+pfe_filter_limit,options.end()); + } + if (pef_filter_only) return; + + SentIdSet fset; + vector locations; + //std::cerr << "Looking up f-phrase: " << options.front()->f_phrase << "\n"; + + locations = f_sa.locateExactPhraseInCorpus(options.front()->f_phrase.c_str()); + if(locations.size()==0) { + cerr<<"No occurrences found!!\n"; + } + for (vector::iterator i=locations.begin(); + i != locations.end(); + ++i) { + fset.insert(i->sentIdInCorpus); + } + size_t cf = fset.size(); + for (std::vector::iterator i=options.begin(); i != options.end(); ++i) { + const std::string& e_phrase = (*i)->e_phrase; + size_t cef=0; + SentIdSet& eset = esets[(*i)->e_phrase]; + if (eset.empty()) { + //std::cerr << "Looking up e-phrase: " << e_phrase << "\n"; + vector locations = e_sa.locateExactPhraseInCorpus(e_phrase.c_str()); + for (vector::iterator i=locations.begin(); i!= locations.end(); ++i) { + TextLenType curSentId = i->sentIdInCorpus; + eset.insert(curSentId); + } + } + size_t ce=eset.size(); + if (ce < cf) { + for (SentIdSet::iterator i=eset.begin(); i != eset.end(); ++i) { + if (fset.find(*i) != fset.end()) cef++; + } + } else { + for (SentIdSet::iterator i=fset.begin(); i != fset.end(); ++i) { + if (eset.find(*i) != eset.end()) cef++; + } + } + double nlp = -log(fisher_exact(cef, cf, ce)); + (*i)->set_cooc_stats(cef, cf, ce, nlp); + if (ce < MINIMUM_SIZE_TO_KEEP) { + esets.erase(e_phrase); + } + } + std::vector::iterator new_end = + std::remove_if(options.begin(), options.end(), NlogSigThresholder(sig_filter_limit)); + nremoved_sigfilter += (options.end() - new_end); + options.erase(new_end,options.end()); +} + +int main(int argc, char * argv[]) +{ + int c; + const char* efile=0; + const char* ffile=0; + int pfe_index = 2; + while ((c = getopt(argc, argv, "cpf:e:i:n:l:")) != -1) { + switch (c) { + case 'e': + efile = optarg; + break; + case 'f': + ffile = optarg; + break; + case 'i': // index of pfe in phrase table + pfe_index = atoi(optarg); + break; + case 'n': // keep only the top n entries in phrase table sorted by p(f|e) (0=all) + pfe_filter_limit = atoi(optarg); + std::cerr << "P(f|e) filter limit: " << pfe_filter_limit << std::endl; + break; + case 'c': + print_cooc_counts = true; + break; + case 'p': + print_neglog_significance = true; + break; + case 'l': + std::cerr << "-l = " << optarg << "\n"; + if (strcmp(optarg,"a+e") == 0) { + sig_filter_limit = ALPHA_PLUS_EPS; + } else if (strcmp(optarg,"a-e") == 0) { + sig_filter_limit = ALPHA_MINUS_EPS; + } else { + char *x; + sig_filter_limit = strtod(optarg, &x); + } + break; + default: + usage(); + } + } + //----------------------------------------------------------------------------- + if (optind != argc || ((!efile || !ffile) && !pef_filter_only)) { + usage(); + } + + //load the indexed corpus with vocabulary(noVoc=false) and with offset(noOffset=false) + if (!pef_filter_only) { + e_sa.loadData_forSearch(efile, false, false); + f_sa.loadData_forSearch(ffile, false, false); + size_t elines = e_sa.returnTotalSentNumber(); + size_t flines = f_sa.returnTotalSentNumber(); + if (elines != flines) { + std::cerr << "Number of lines in e-corpus != number of lines in f-corpus!\n"; + usage(); + } else { + std::cerr << "Training corpus: " << elines << " lines\n"; + num_lines = elines; + } + p_111 = -log(fisher_exact(1,1,1)); + std::cerr << "\\alpha = " << p_111 << "\n"; + if (sig_filter_limit == ALPHA_MINUS_EPS) { + sig_filter_limit = p_111 - 0.001; + } else if (sig_filter_limit == ALPHA_PLUS_EPS) { + sig_filter_limit = p_111 + 0.001; + } + std::cerr << "Sig filter threshold is = " << sig_filter_limit << "\n"; + } else { + std::cerr << "Filtering using P(e|f) only. n=" << pfe_filter_limit << std::endl; + } + + char tmpString[10000]; + std::string prev = ""; + std::vector options; + size_t pt_lines = 0; + while(!cin.eof()) { + cin.getline(tmpString,10000,'\n'); + if(++pt_lines%10000==0) { + std::cerr << "."; + if(pt_lines%500000==0) std::cerr << "[n:"<0) { + PTEntry* pp = new PTEntry(tmpString, pfe_index); + if (prev != pp->f_phrase) { + prev = pp->f_phrase; + + if (!options.empty()) { // always true after first line + compute_cooc_stats_and_filter(options); + } + for (std::vector::iterator i=options.begin(); i != options.end(); ++i) { + std::cout << **i << std::endl; + delete *i; + } + options.clear(); + options.push_back(pp); + + } else { + options.push_back(pp); + } + // for(int i=0;i/moses--1..rpm. + +For example building on a 64 bit Intel architecture, and building v1.0 the RPM would be called moses-1.0-1.x86_64.rpm. + + +Building a Debian package +------------------------- + +The Alien tool converts RPM packages to Debian packages. If a Debian package is required then follow the instructions on the following web-page: + +https://help.ubuntu.com/community/RPM/AlienHowto diff --git a/mosesdecoder/contrib/rpm/build_source.sh b/mosesdecoder/contrib/rpm/build_source.sh new file mode 100644 index 0000000000000000000000000000000000000000..e1d83f2879ea99c29bb9ebc9baa6645fbcb6fed3 --- /dev/null +++ b/mosesdecoder/contrib/rpm/build_source.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +BRANCH="master" +BOOST="/usr" +declare -i NO_RPM_BUILD=0 +declare -i RELEASE=1 +declare -r RPM_VERSION_TAG="___RPM_VERSION__" +declare -r RPM_RELEASE_TAG="___RPM_RELEASE__" +declare -r BOOST_TAG="___BOOST_LOCATION__" + +function usage() { + echo "`basename $0` -r [Moses Git repo] -b [Moses Git branch: default ${BRANCH}] -v [RPM version] -l [RPM release: default ${RELEASE}] -t [Boost install: default ${BOOST}]" + exit 1 +} + +if [ $# -lt 4 ]; then + usage +fi + +while getopts r:b:t:v:l:nh OPTION +do + case "$OPTION" in + r) REPO="${OPTARG}";; + b) BRANCH="${OPTARG}";; + t) BOOST="${OPTARG}";; + v) VERSION="${OPTARG}";; + l) RELEASE="${OPTARG}";; + n) NO_RPM_BUILD=1;; + [h\?]) usage;; + esac +done + +if [ ! -d ./rpmbuild ]; then + echo "RPM build directory not in current working direcotry" + exit 1 +fi + +declare -r MOSES_DIR="moses-${VERSION}" +git clone ${REPO} ${MOSES_DIR} +if [ $? -ne 0 ]; then + echo "Failed to clone Git repository ${REPO}" + exit 3 +fi + +cd ${MOSES_DIR} + +git checkout ${BRANCH} +if [ $? -ne 0 ]; then + echo "Failed to checkout branch ${BRANCH}" + exit 3 +fi + +cd .. + +tar -cf moses-${VERSION}.tar ${MOSES_DIR} +gzip -f9 moses-${VERSION}.tar + +if [ ${NO_RPM_BUILD} -eq 0 ]; then + if [ ! -d ${HOME}/rpmbuild/SPECS ]; then + mkdir -p ${HOME}/rpmbuild/SPECS + fi + ESC_BOOST=`echo ${BOOST} | gawk '{gsub(/\//, "\\\\/"); print}'` + eval sed -e \"s/${RPM_VERSION_TAG}/${VERSION}/\" -e \"s/${RPM_RELEASE_TAG}/${RELEASE}/\" -e \"s/${BOOST_TAG}/${ESC_BOOST}/\" ./rpmbuild/SPECS/moses.spec > ${HOME}/rpmbuild/SPECS/moses.spec + if [ ! -d ${HOME}/rpmbuild/SOURCES ]; then + mkdir -p ${HOME}/rpmbuild/SOURCES + fi + mv moses-${VERSION}.tar.gz ${HOME}/rpmbuild/SOURCES +fi + +rm -Rf ${MOSES_DIR} diff --git a/mosesdecoder/contrib/rpm/rpmbuild/SPECS/moses.spec b/mosesdecoder/contrib/rpm/rpmbuild/SPECS/moses.spec new file mode 100644 index 0000000000000000000000000000000000000000..fc4c2f9716094a8d27fd9eff3b12d740d27618d6 --- /dev/null +++ b/mosesdecoder/contrib/rpm/rpmbuild/SPECS/moses.spec @@ -0,0 +1,82 @@ +%define name moses +%define version ___RPM_VERSION__ +%define release ___RPM_RELEASE__ + +Name: %{name} +Summary: Moses is a statistical machine translation system that allows you to automatically train translation models for any language pair. +Version: %{version} +Release: %{release} +URL: http://www.statmt.org/%{name}-%{version}/ +Source0: %{name}-%{version}.tar.gz +License: LGPL +Group: Development/Tools +Vendor: Capita Translation and Interpreting +Packager: Ian Johnson +Requires: python >= 2.6, perl >= 5 +Prefix: /opt +BuildRoot: %{_builddir}/%{name}-%{version}-%{release} +%description +Moses is a statistical machine translation system that allows you to automatically train translation models for any language pair. All you need is a collection of translated texts (parallel corpus). An efficient search algorithm finds quickly the highest probability translation among the exponential number of choices. +%prep +%setup -q + +mkdir -p $RPM_BUILD_ROOT/opt/%{name}-%{version}/giza++-v1.0.7 + +wget -O $RPM_BUILD_DIR/irstlm-5.70.04.tgz http://moses-suite.googlecode.com/files/irstlm-5.70.04.tgz +wget -O $RPM_BUILD_DIR/giza-pp-v1.0.7.tgz http://moses-suite.googlecode.com/files/giza-pp-v1.0.7.tar.gz + +cd $RPM_BUILD_DIR + +tar -zxf irstlm-5.70.04.tgz +tar -zxf giza-pp-v1.0.7.tgz + +cd irstlm-5.70.04 +bash regenerate-makefiles.sh --force +./configure --prefix $RPM_BUILD_ROOT/opt/%{name}-%{version}/irstlm-5.70.04 +make +make install + +cd ../giza-pp +make +cp $RPM_BUILD_DIR/giza-pp/GIZA++-v2/GIZA++ $RPM_BUILD_DIR/giza-pp/GIZA++-v2/snt2cooc.out $RPM_BUILD_DIR/giza-pp/mkcls-v2/mkcls $RPM_BUILD_ROOT/opt/%{name}-%{version}/giza++-v1.0.7 +%build +./bjam --with-boost=___BOOST_LOCATION__ --with-irstlm=$RPM_BUILD_ROOT/opt/%{name}-%{version}/irstlm-5.70.04 --with-giza=$RPM_BUILD_ROOT/opt/%{name}-%{version}/giza++-v1.0.7 -j2 +%install +mkdir -p $RPM_BUILD_ROOT/opt/%{name}-%{version}/scripts +cp -R bin $RPM_BUILD_ROOT/opt/%{name}-%{version} +cp -R scripts/OSM $RPM_BUILD_ROOT/opt/%{name}-%{version}/scripts +cp -R scripts/Transliteration $RPM_BUILD_ROOT/opt/%{name}-%{version}/scripts +cp -R scripts/analysis $RPM_BUILD_ROOT/opt/%{name}-%{version}/scripts +cp -R scripts/ems $RPM_BUILD_ROOT/opt/%{name}-%{version}/scripts +cp -R scripts/generic $RPM_BUILD_ROOT/opt/%{name}-%{version}/scripts +cp -R scripts/other $RPM_BUILD_ROOT/opt/%{name}-%{version}/scripts +cp -R scripts/recaser $RPM_BUILD_ROOT/opt/%{name}-%{version}/scripts +cp -R scripts/share $RPM_BUILD_ROOT/opt/%{name}-%{version}/scripts +cp -R scripts/tokenizer $RPM_BUILD_ROOT/opt/%{name}-%{version}/scripts +cp -R scripts/training $RPM_BUILD_ROOT/opt/%{name}-%{version}/scripts +%clean +%files +%defattr(-,root,root) +/opt/%{name}-%{version}/bin/* +/opt/%{name}-%{version}/scripts/OSM/* +/opt/%{name}-%{version}/scripts/Transliteration/* +/opt/%{name}-%{version}/scripts/analysis/* +/opt/%{name}-%{version}/scripts/ems/* +/opt/%{name}-%{version}/scripts/generic/* +/opt/%{name}-%{version}/scripts/other/* +/opt/%{name}-%{version}/scripts/recaser/* +/opt/%{name}-%{version}/scripts/share/* +/opt/%{name}-%{version}/scripts/tokenizer/* +/opt/%{name}-%{version}/scripts/training/* +/opt/%{name}-%{version}/irstlm-5.70.04/* +/opt/%{name}-%{version}/giza++-v1.0.7/* +%pre +if [ "$1" = "1" ]; then +elif [ "$1" = "2" ]; then + rm $RPM_INSTALL_PREFIX/%{name} 2>/dev/null +fi +%post +ln -s $RPM_INSTALL_PREFIX/%{name}-%{version} $RPM_INSTALL_PREFIX/%{name} +%postun +rm -Rf $RPM_INSTALL_PREFIX/%{name}-%{version} 2>/dev/null +rm $RPM_INSTALL_PREFIX/%{name} 2>/dev/null diff --git a/mosesdecoder/contrib/synlm/hhmm/LICENSE b/mosesdecoder/contrib/synlm/hhmm/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..25b5b709763f867db8430de49117b38e1b047eaa --- /dev/null +++ b/mosesdecoder/contrib/synlm/hhmm/LICENSE @@ -0,0 +1,695 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + + +MOSES EXCEPTION TO THE GPL + +Certain source files distributed by ModelBlocks developers are +subject to the following clarification and special exception to the GPL, but +only where ModelBlocks developers have expressly included in the particular source file's header +the words "ModelBlocks developers designate this particular file as subject to the "Moses" +exception as provided by ModelBlocks developers in the LICENSE file that accompanies this code." + + Linking this library statically or dynamically with other modules is making + a combined work based on this library. Thus, the terms and conditions of + the GNU General Public License cover the whole combination. + + As a special exception, the copyright holders of this library give you + permission to compile and link this library with Moses modules to produce an + executable, and to copy and distribute the resulting executable under the terms + of the Moses license, provided that you also meet the terms and conditions of that license. + If you modify this library, you may extend this exception to your version of + the library, but you are not obligated to do so. If you do not wish to do + so, delete this exception statement from your version. diff --git a/mosesdecoder/contrib/synlm/hhmm/rvtl/include/nl-array.h b/mosesdecoder/contrib/synlm/hhmm/rvtl/include/nl-array.h new file mode 100644 index 0000000000000000000000000000000000000000..6c6e1bb5fa47b5883350a78fecb9b538377f4e30 --- /dev/null +++ b/mosesdecoder/contrib/synlm/hhmm/rvtl/include/nl-array.h @@ -0,0 +1,163 @@ +/////////////////////////////////////////////////////////////////////////////// +// // +// This file is part of ModelBlocks. Copyright 2009, ModelBlocks developers. // +// // +// ModelBlocks is free software: you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation, either version 3 of the License, or // +// (at your option) any later version. // +// // +// ModelBlocks is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with ModelBlocks. If not, see . // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + + +#ifndef _NL_ARRAY__ +#define _NL_ARRAY__ + +#include +#include +using namespace std; + +//////////////////////////////////////////////////////////////////////////////// + +template +class Array { + private: + + T* pt; + unsigned int iNextOpen; + unsigned int iCapacity; + + static const T tDummy; + + public: + + class const_iterator; + class iterator : public pair*,int> { + typedef pair*,int> parent; + public: + iterator ( Array* pat, int i ) : parent(pat,i) { } + bool operator== ( const iterator& it ) { return (parent::first==it.first && parent::second==it.second); } + bool operator== ( const const_iterator& it ) { return (parent::first==it.first && parent::second==it.second); } + bool operator!= ( const iterator& it ) { return (parent::first!=it.first || parent::second!=it.second); } + bool operator!= ( const const_iterator& it ) { return (parent::first!=it.first || parent::second!=it.second); } + T* operator-> ( ) { return &parent::first->set(parent::second); } + iterator& operator++ ( ) { ++parent::second; return *this; } + iterator next ( ) { return iterator ( parent::first, parent::second+1 ); } + }; + class const_iterator : public pair*,int> { + typedef pair*,int> parent; + public: + const_iterator ( const iterator& it ) : parent(it) { } + const_iterator ( const Array* pat, int i ) : parent(pat,i) { } + bool operator== ( const iterator& it ) { return (parent::first==it.first && parent::second==it.second); } + bool operator== ( const const_iterator& it ) { return (parent::first==it.first && parent::second==it.second); } + bool operator!= ( const iterator& it ) { return (parent::first!=it.first || parent::second!=it.second); } + bool operator!= ( const const_iterator& it ) { return (parent::first!=it.first || parent::second!=it.second); } + const T* operator-> ( ) { return &parent::first->get(parent::second); } + const_iterator& operator++ ( ) { ++parent::second; return *this; } + const_iterator next ( ) { return const_iterator ( parent::first, parent::second+1 ); } + }; + + // Constructor / destructor methods... + Array ( ) : pt(new T[1]), iNextOpen(0), iCapacity(1) { } + Array ( unsigned int i ) : pt(new T[i]), iNextOpen(0), iCapacity(i) { } + Array ( unsigned int i, const T& t ) : pt(new T[i]), iNextOpen(0), iCapacity(i) { for(unsigned int j=0; j& at ) : pt(new T[at.iCapacity]), iNextOpen(at.iNextOpen), iCapacity(at.iCapacity) + { for(unsigned int i=0; i& operator= ( const Array& at ) { delete[] pt; pt=new T[at.iCapacity]; iNextOpen=at.iNextOpen; iCapacity=at.iCapacity; + for(unsigned int i=0; i& ensureCapacity ( unsigned int i ) { + if ( i>=iCapacity ) { + T* ptTmp = pt; + iCapacity = (i=iCapacity) return tDummy; /*assert(i=iNextOpen)iNextOpen=i+1; return pt[i]; } + + // Aliasing methods... + T* c_array ( ) { return pt; } // NOTE: now it's risky! + const T* c_array ( ) const { return pt; } + + // Extraction methods... + unsigned int size() const { return iNextOpen; } + unsigned int getSize() const { return iNextOpen; } + bool operator== ( const Array& a ) const { + if ( size() != a.size() ) return false; + for ( unsigned int i=0; i& a ) const { return !(*this==a); } + + // Input / output methods... + friend ostream& operator<< ( ostream& os, const Array& at ) { for(unsigned int i=0;i const T Array::tDummy = T(); + + +//////////////////////////////////////////////////////////////////////////////// + +template +class SubArray { + private: + T* pat; + unsigned int length; + public: + // Constructor / destructor methods... + SubArray ( ) : pat(NULL), length(0) { } + SubArray ( Array& at ) : pat(&at.set(0)), length(at.size()) { } + SubArray ( Array& at, unsigned int i ) : pat(&at.set(i)), length(at.size()-i) { } + SubArray ( Array& at, unsigned int i, unsigned int j ) : pat(&at.set(i)), length(j-i) { } + SubArray ( SubArray& at, unsigned int i ) : pat(&at.set(i)), length(at.size()-i) { } + SubArray ( SubArray& at, unsigned int i, unsigned int j ) : pat(&at.set(i)), length(j-i) { } + // Indexing methods... + const T& operator[] ( unsigned int i ) const { return get(i); } + T& operator[] ( unsigned int i ) { return set(i); } + const T& get ( unsigned int i ) const { assert(i. // +// // +// ModelBlocks developers designate this particular file as subject to // +// the "Moses" exception as provided by ModelBlocks developers in // +// the LICENSE file that accompanies this code. // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _NL_MIXTURE__ +#define _NL_MIXTURE__ + +#include "nl-randvar.h" +#include "nl-list.h" + +//////////////////////////////////////////////////////////////////////////////// +// +// Mixture Model +// +//////////////////////////////////////////////////////////////////////////////// + +template