diff --git a/FacebookAI/xlm-roberta-large/onnx/sentencepiece.bpe.model b/FacebookAI/xlm-roberta-large/onnx/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..7a3f40a75f870bc1f21700cd414dc2acc431583c --- /dev/null +++ b/FacebookAI/xlm-roberta-large/onnx/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865 +size 5069051 diff --git a/mosesdecoder/contrib/memscore/Makefile.am b/mosesdecoder/contrib/memscore/Makefile.am new file mode 100644 index 0000000000000000000000000000000000000000..1cf79f2b35480bfafa025e01cd7e37b2cd2bc84a --- /dev/null +++ b/mosesdecoder/contrib/memscore/Makefile.am @@ -0,0 +1,24 @@ +# memscore - in-memory phrase scoring for Statistical Machine Translation +# Christian Hardmeier, FBK-irst, Trento, 2010 +# $Id$ + +## Process this file with automake to produce Makefile.in + +ACLOCAL_AMFLAGS = -I m4 + +AUTOMAKE_OPTIONS = foreign +AM_CXXFLAGS = $(BOOST_CPPFLAGS) -Wall -ffast-math -ftrapping-math -fomit-frame-pointer + +bin_PROGRAMS = memscore +memscore_SOURCES = datastorage.h memscore.h phrasetable.h scorer.h scorer-impl.h statistic.h timestamp.h \ + phrasetable.cpp memscore.cpp scorer.cpp lexdecom.cpp lexdecom.h + +if IRSTLM +memscore_SOURCES += phraselm.cpp phraselm.h +endif +if CHANNEL_SCORER +memscore_SOURCES += channel-scorer.cpp channel-scorer.h +endif + + +memscore_LDADD = $(IRSTLM_LIBS) $(GSL_LIBS) diff --git a/mosesdecoder/contrib/memscore/aclocal.m4 b/mosesdecoder/contrib/memscore/aclocal.m4 new file mode 100644 index 0000000000000000000000000000000000000000..fcc48b9279398869b574c08e50854f4fc1f81c54 --- /dev/null +++ b/mosesdecoder/contrib/memscore/aclocal.m4 @@ -0,0 +1,851 @@ +# generated automatically by aclocal 1.9.6 -*- Autoconf -*- + +# Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, +# 2005 Free Software Foundation, Inc. +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +# Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# AM_AUTOMAKE_VERSION(VERSION) +# ---------------------------- +# Automake X.Y traces this macro to ensure aclocal.m4 has been +# generated from the m4 files accompanying Automake X.Y. +AC_DEFUN([AM_AUTOMAKE_VERSION], [am__api_version="1.9"]) + +# AM_SET_CURRENT_AUTOMAKE_VERSION +# ------------------------------- +# Call AM_AUTOMAKE_VERSION so it can be traced. +# This function is AC_REQUIREd by AC_INIT_AUTOMAKE. +AC_DEFUN([AM_SET_CURRENT_AUTOMAKE_VERSION], + [AM_AUTOMAKE_VERSION([1.9.6])]) + +# AM_AUX_DIR_EXPAND -*- Autoconf -*- + +# Copyright (C) 2001, 2003, 2005 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# For projects using AC_CONFIG_AUX_DIR([foo]), Autoconf sets +# $ac_aux_dir to `$srcdir/foo'. In other projects, it is set to +# `$srcdir', `$srcdir/..', or `$srcdir/../..'. +# +# Of course, Automake must honor this variable whenever it calls a +# tool from the auxiliary directory. The problem is that $srcdir (and +# therefore $ac_aux_dir as well) can be either absolute or relative, +# depending on how configure is run. This is pretty annoying, since +# it makes $ac_aux_dir quite unusable in subdirectories: in the top +# source directory, any form will work fine, but in subdirectories a +# relative path needs to be adjusted first. +# +# $ac_aux_dir/missing +# fails when called from a subdirectory if $ac_aux_dir is relative +# $top_srcdir/$ac_aux_dir/missing +# fails if $ac_aux_dir is absolute, +# fails when called from a subdirectory in a VPATH build with +# a relative $ac_aux_dir +# +# The reason of the latter failure is that $top_srcdir and $ac_aux_dir +# are both prefixed by $srcdir. In an in-source build this is usually +# harmless because $srcdir is `.', but things will broke when you +# start a VPATH build or use an absolute $srcdir. +# +# So we could use something similar to $top_srcdir/$ac_aux_dir/missing, +# iff we strip the leading $srcdir from $ac_aux_dir. That would be: +# am_aux_dir='\$(top_srcdir)/'`expr "$ac_aux_dir" : "$srcdir//*\(.*\)"` +# and then we would define $MISSING as +# MISSING="\${SHELL} $am_aux_dir/missing" +# This will work as long as MISSING is not called from configure, because +# unfortunately $(top_srcdir) has no meaning in configure. +# However there are other variables, like CC, which are often used in +# configure, and could therefore not use this "fixed" $ac_aux_dir. +# +# Another solution, used here, is to always expand $ac_aux_dir to an +# absolute PATH. The drawback is that using absolute paths prevent a +# configured tree to be moved without reconfiguration. + +AC_DEFUN([AM_AUX_DIR_EXPAND], +[dnl Rely on autoconf to set up CDPATH properly. +AC_PREREQ([2.50])dnl +# expand $ac_aux_dir to an absolute path +am_aux_dir=`cd $ac_aux_dir && pwd` +]) + +# AM_CONDITIONAL -*- Autoconf -*- + +# Copyright (C) 1997, 2000, 2001, 2003, 2004, 2005 +# Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 7 + +# AM_CONDITIONAL(NAME, SHELL-CONDITION) +# ------------------------------------- +# Define a conditional. +AC_DEFUN([AM_CONDITIONAL], +[AC_PREREQ(2.52)dnl + ifelse([$1], [TRUE], [AC_FATAL([$0: invalid condition: $1])], + [$1], [FALSE], [AC_FATAL([$0: invalid condition: $1])])dnl +AC_SUBST([$1_TRUE]) +AC_SUBST([$1_FALSE]) +if $2; then + $1_TRUE= + $1_FALSE='#' +else + $1_TRUE='#' + $1_FALSE= +fi +AC_CONFIG_COMMANDS_PRE( +[if test -z "${$1_TRUE}" && test -z "${$1_FALSE}"; then + AC_MSG_ERROR([[conditional "$1" was never defined. +Usually this means the macro was only invoked conditionally.]]) +fi])]) + + +# Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005 +# Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 8 + +# There are a few dirty hacks below to avoid letting `AC_PROG_CC' be +# written in clear, in which case automake, when reading aclocal.m4, +# will think it sees a *use*, and therefore will trigger all it's +# C support machinery. Also note that it means that autoscan, seeing +# CC etc. in the Makefile, will ask for an AC_PROG_CC use... + + +# _AM_DEPENDENCIES(NAME) +# ---------------------- +# See how the compiler implements dependency checking. +# NAME is "CC", "CXX", "GCJ", or "OBJC". +# We try a few techniques and use that to set a single cache variable. +# +# We don't AC_REQUIRE the corresponding AC_PROG_CC since the latter was +# modified to invoke _AM_DEPENDENCIES(CC); we would have a circular +# dependency, and given that the user is not expected to run this macro, +# just rely on AC_PROG_CC. +AC_DEFUN([_AM_DEPENDENCIES], +[AC_REQUIRE([AM_SET_DEPDIR])dnl +AC_REQUIRE([AM_OUTPUT_DEPENDENCY_COMMANDS])dnl +AC_REQUIRE([AM_MAKE_INCLUDE])dnl +AC_REQUIRE([AM_DEP_TRACK])dnl + +ifelse([$1], CC, [depcc="$CC" am_compiler_list=], + [$1], CXX, [depcc="$CXX" am_compiler_list=], + [$1], OBJC, [depcc="$OBJC" am_compiler_list='gcc3 gcc'], + [$1], GCJ, [depcc="$GCJ" am_compiler_list='gcc3 gcc'], + [depcc="$$1" am_compiler_list=]) + +AC_CACHE_CHECK([dependency style of $depcc], + [am_cv_$1_dependencies_compiler_type], +[if test -z "$AMDEP_TRUE" && test -f "$am_depcomp"; then + # We make a subdir and do the tests there. Otherwise we can end up + # making bogus files that we don't know about and never remove. For + # instance it was reported that on HP-UX the gcc test will end up + # making a dummy file named `D' -- because `-MD' means `put the output + # in D'. + mkdir conftest.dir + # Copy depcomp to subdir because otherwise we won't find it if we're + # using a relative directory. + cp "$am_depcomp" conftest.dir + cd conftest.dir + # We will build objects and dependencies in a subdirectory because + # it helps to detect inapplicable dependency modes. For instance + # both Tru64's cc and ICC support -MD to output dependencies as a + # side effect of compilation, but ICC will put the dependencies in + # the current directory while Tru64 will put them in the object + # directory. + mkdir sub + + am_cv_$1_dependencies_compiler_type=none + if test "$am_compiler_list" = ""; then + am_compiler_list=`sed -n ['s/^#*\([a-zA-Z0-9]*\))$/\1/p'] < ./depcomp` + fi + for depmode in $am_compiler_list; do + # Setup a source with many dependencies, because some compilers + # like to wrap large dependency lists on column 80 (with \), and + # we should not choose a depcomp mode which is confused by this. + # + # We need to recreate these files for each test, as the compiler may + # overwrite some of them when testing with obscure command lines. + # This happens at least with the AIX C compiler. + : > sub/conftest.c + for i in 1 2 3 4 5 6; do + echo '#include "conftst'$i'.h"' >> sub/conftest.c + # Using `: > sub/conftst$i.h' creates only sub/conftst1.h with + # Solaris 8's {/usr,}/bin/sh. + touch sub/conftst$i.h + done + echo "${am__include} ${am__quote}sub/conftest.Po${am__quote}" > confmf + + case $depmode in + nosideeffect) + # after this tag, mechanisms are not by side-effect, so they'll + # only be used when explicitly requested + if test "x$enable_dependency_tracking" = xyes; then + continue + else + break + fi + ;; + none) break ;; + esac + # We check with `-c' and `-o' for the sake of the "dashmstdout" + # mode. It turns out that the SunPro C++ compiler does not properly + # handle `-M -o', and we need to detect this. + if depmode=$depmode \ + source=sub/conftest.c object=sub/conftest.${OBJEXT-o} \ + depfile=sub/conftest.Po tmpdepfile=sub/conftest.TPo \ + $SHELL ./depcomp $depcc -c -o sub/conftest.${OBJEXT-o} sub/conftest.c \ + >/dev/null 2>conftest.err && + grep sub/conftst6.h sub/conftest.Po > /dev/null 2>&1 && + grep sub/conftest.${OBJEXT-o} sub/conftest.Po > /dev/null 2>&1 && + ${MAKE-make} -s -f confmf > /dev/null 2>&1; then + # icc doesn't choke on unknown options, it will just issue warnings + # or remarks (even with -Werror). So we grep stderr for any message + # that says an option was ignored or not supported. + # When given -MP, icc 7.0 and 7.1 complain thusly: + # icc: Command line warning: ignoring option '-M'; no argument required + # The diagnosis changed in icc 8.0: + # icc: Command line remark: option '-MP' not supported + if (grep 'ignoring option' conftest.err || + grep 'not supported' conftest.err) >/dev/null 2>&1; then :; else + am_cv_$1_dependencies_compiler_type=$depmode + break + fi + fi + done + + cd .. + rm -rf conftest.dir +else + am_cv_$1_dependencies_compiler_type=none +fi +]) +AC_SUBST([$1DEPMODE], [depmode=$am_cv_$1_dependencies_compiler_type]) +AM_CONDITIONAL([am__fastdep$1], [ + test "x$enable_dependency_tracking" != xno \ + && test "$am_cv_$1_dependencies_compiler_type" = gcc3]) +]) + + +# AM_SET_DEPDIR +# ------------- +# Choose a directory name for dependency files. +# This macro is AC_REQUIREd in _AM_DEPENDENCIES +AC_DEFUN([AM_SET_DEPDIR], +[AC_REQUIRE([AM_SET_LEADING_DOT])dnl +AC_SUBST([DEPDIR], ["${am__leading_dot}deps"])dnl +]) + + +# AM_DEP_TRACK +# ------------ +AC_DEFUN([AM_DEP_TRACK], +[AC_ARG_ENABLE(dependency-tracking, +[ --disable-dependency-tracking speeds up one-time build + --enable-dependency-tracking do not reject slow dependency extractors]) +if test "x$enable_dependency_tracking" != xno; then + am_depcomp="$ac_aux_dir/depcomp" + AMDEPBACKSLASH='\' +fi +AM_CONDITIONAL([AMDEP], [test "x$enable_dependency_tracking" != xno]) +AC_SUBST([AMDEPBACKSLASH]) +]) + +# Generate code to set up dependency tracking. -*- Autoconf -*- + +# Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005 +# Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +#serial 3 + +# _AM_OUTPUT_DEPENDENCY_COMMANDS +# ------------------------------ +AC_DEFUN([_AM_OUTPUT_DEPENDENCY_COMMANDS], +[for mf in $CONFIG_FILES; do + # Strip MF so we end up with the name of the file. + mf=`echo "$mf" | sed -e 's/:.*$//'` + # Check whether this is an Automake generated Makefile or not. + # We used to match only the files named `Makefile.in', but + # some people rename them; so instead we look at the file content. + # Grep'ing the first line is not enough: some people post-process + # each Makefile.in and add a new line on top of each file to say so. + # So let's grep whole file. + if grep '^#.*generated by automake' $mf > /dev/null 2>&1; then + dirpart=`AS_DIRNAME("$mf")` + else + continue + fi + # Extract the definition of DEPDIR, am__include, and am__quote + # from the Makefile without running `make'. + DEPDIR=`sed -n 's/^DEPDIR = //p' < "$mf"` + test -z "$DEPDIR" && continue + am__include=`sed -n 's/^am__include = //p' < "$mf"` + test -z "am__include" && continue + am__quote=`sed -n 's/^am__quote = //p' < "$mf"` + # When using ansi2knr, U may be empty or an underscore; expand it + U=`sed -n 's/^U = //p' < "$mf"` + # Find all dependency output files, they are included files with + # $(DEPDIR) in their names. We invoke sed twice because it is the + # simplest approach to changing $(DEPDIR) to its actual value in the + # expansion. + for file in `sed -n " + s/^$am__include $am__quote\(.*(DEPDIR).*\)$am__quote"'$/\1/p' <"$mf" | \ + sed -e 's/\$(DEPDIR)/'"$DEPDIR"'/g' -e 's/\$U/'"$U"'/g'`; do + # Make sure the directory exists. + test -f "$dirpart/$file" && continue + fdir=`AS_DIRNAME(["$file"])` + AS_MKDIR_P([$dirpart/$fdir]) + # echo "creating $dirpart/$file" + echo '# dummy' > "$dirpart/$file" + done +done +])# _AM_OUTPUT_DEPENDENCY_COMMANDS + + +# AM_OUTPUT_DEPENDENCY_COMMANDS +# ----------------------------- +# This macro should only be invoked once -- use via AC_REQUIRE. +# +# This code is only required when automatic dependency tracking +# is enabled. FIXME. This creates each `.P' file that we will +# need in order to bootstrap the dependency handling code. +AC_DEFUN([AM_OUTPUT_DEPENDENCY_COMMANDS], +[AC_CONFIG_COMMANDS([depfiles], + [test x"$AMDEP_TRUE" != x"" || _AM_OUTPUT_DEPENDENCY_COMMANDS], + [AMDEP_TRUE="$AMDEP_TRUE" ac_aux_dir="$ac_aux_dir"]) +]) + +# Do all the work for Automake. -*- Autoconf -*- + +# Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005 +# Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 12 + +# This macro actually does too much. Some checks are only needed if +# your package does certain things. But this isn't really a big deal. + +# AM_INIT_AUTOMAKE(PACKAGE, VERSION, [NO-DEFINE]) +# AM_INIT_AUTOMAKE([OPTIONS]) +# ----------------------------------------------- +# The call with PACKAGE and VERSION arguments is the old style +# call (pre autoconf-2.50), which is being phased out. PACKAGE +# and VERSION should now be passed to AC_INIT and removed from +# the call to AM_INIT_AUTOMAKE. +# We support both call styles for the transition. After +# the next Automake release, Autoconf can make the AC_INIT +# arguments mandatory, and then we can depend on a new Autoconf +# release and drop the old call support. +AC_DEFUN([AM_INIT_AUTOMAKE], +[AC_PREREQ([2.58])dnl +dnl Autoconf wants to disallow AM_ names. We explicitly allow +dnl the ones we care about. +m4_pattern_allow([^AM_[A-Z]+FLAGS$])dnl +AC_REQUIRE([AM_SET_CURRENT_AUTOMAKE_VERSION])dnl +AC_REQUIRE([AC_PROG_INSTALL])dnl +# test to see if srcdir already configured +if test "`cd $srcdir && pwd`" != "`pwd`" && + test -f $srcdir/config.status; then + AC_MSG_ERROR([source directory already configured; run "make distclean" there first]) +fi + +# test whether we have cygpath +if test -z "$CYGPATH_W"; then + if (cygpath --version) >/dev/null 2>/dev/null; then + CYGPATH_W='cygpath -w' + else + CYGPATH_W=echo + fi +fi +AC_SUBST([CYGPATH_W]) + +# Define the identity of the package. +dnl Distinguish between old-style and new-style calls. +m4_ifval([$2], +[m4_ifval([$3], [_AM_SET_OPTION([no-define])])dnl + AC_SUBST([PACKAGE], [$1])dnl + AC_SUBST([VERSION], [$2])], +[_AM_SET_OPTIONS([$1])dnl + AC_SUBST([PACKAGE], ['AC_PACKAGE_TARNAME'])dnl + AC_SUBST([VERSION], ['AC_PACKAGE_VERSION'])])dnl + +_AM_IF_OPTION([no-define],, +[AC_DEFINE_UNQUOTED(PACKAGE, "$PACKAGE", [Name of package]) + AC_DEFINE_UNQUOTED(VERSION, "$VERSION", [Version number of package])])dnl + +# Some tools Automake needs. +AC_REQUIRE([AM_SANITY_CHECK])dnl +AC_REQUIRE([AC_ARG_PROGRAM])dnl +AM_MISSING_PROG(ACLOCAL, aclocal-${am__api_version}) +AM_MISSING_PROG(AUTOCONF, autoconf) +AM_MISSING_PROG(AUTOMAKE, automake-${am__api_version}) +AM_MISSING_PROG(AUTOHEADER, autoheader) +AM_MISSING_PROG(MAKEINFO, makeinfo) +AM_PROG_INSTALL_SH +AM_PROG_INSTALL_STRIP +AC_REQUIRE([AM_PROG_MKDIR_P])dnl +# We need awk for the "check" target. The system "awk" is bad on +# some platforms. +AC_REQUIRE([AC_PROG_AWK])dnl +AC_REQUIRE([AC_PROG_MAKE_SET])dnl +AC_REQUIRE([AM_SET_LEADING_DOT])dnl +_AM_IF_OPTION([tar-ustar], [_AM_PROG_TAR([ustar])], + [_AM_IF_OPTION([tar-pax], [_AM_PROG_TAR([pax])], + [_AM_PROG_TAR([v7])])]) +_AM_IF_OPTION([no-dependencies],, +[AC_PROVIDE_IFELSE([AC_PROG_CC], + [_AM_DEPENDENCIES(CC)], + [define([AC_PROG_CC], + defn([AC_PROG_CC])[_AM_DEPENDENCIES(CC)])])dnl +AC_PROVIDE_IFELSE([AC_PROG_CXX], + [_AM_DEPENDENCIES(CXX)], + [define([AC_PROG_CXX], + defn([AC_PROG_CXX])[_AM_DEPENDENCIES(CXX)])])dnl +]) +]) + + +# When config.status generates a header, we must update the stamp-h file. +# This file resides in the same directory as the config header +# that is generated. The stamp files are numbered to have different names. + +# Autoconf calls _AC_AM_CONFIG_HEADER_HOOK (when defined) in the +# loop where config.status creates the headers, so we can generate +# our stamp files there. +AC_DEFUN([_AC_AM_CONFIG_HEADER_HOOK], +[# Compute $1's index in $config_headers. +_am_stamp_count=1 +for _am_header in $config_headers :; do + case $_am_header in + $1 | $1:* ) + break ;; + * ) + _am_stamp_count=`expr $_am_stamp_count + 1` ;; + esac +done +echo "timestamp for $1" >`AS_DIRNAME([$1])`/stamp-h[]$_am_stamp_count]) + +# Copyright (C) 2001, 2003, 2005 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# AM_PROG_INSTALL_SH +# ------------------ +# Define $install_sh. +AC_DEFUN([AM_PROG_INSTALL_SH], +[AC_REQUIRE([AM_AUX_DIR_EXPAND])dnl +install_sh=${install_sh-"$am_aux_dir/install-sh"} +AC_SUBST(install_sh)]) + +# Copyright (C) 2003, 2005 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 2 + +# Check whether the underlying file-system supports filenames +# with a leading dot. For instance MS-DOS doesn't. +AC_DEFUN([AM_SET_LEADING_DOT], +[rm -rf .tst 2>/dev/null +mkdir .tst 2>/dev/null +if test -d .tst; then + am__leading_dot=. +else + am__leading_dot=_ +fi +rmdir .tst 2>/dev/null +AC_SUBST([am__leading_dot])]) + +# Check to see how 'make' treats includes. -*- Autoconf -*- + +# Copyright (C) 2001, 2002, 2003, 2005 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 3 + +# AM_MAKE_INCLUDE() +# ----------------- +# Check to see how make treats includes. +AC_DEFUN([AM_MAKE_INCLUDE], +[am_make=${MAKE-make} +cat > confinc << 'END' +am__doit: + @echo done +.PHONY: am__doit +END +# If we don't find an include directive, just comment out the code. +AC_MSG_CHECKING([for style of include used by $am_make]) +am__include="#" +am__quote= +_am_result=none +# First try GNU make style include. +echo "include confinc" > confmf +# We grep out `Entering directory' and `Leaving directory' +# messages which can occur if `w' ends up in MAKEFLAGS. +# In particular we don't look at `^make:' because GNU make might +# be invoked under some other name (usually "gmake"), in which +# case it prints its new name instead of `make'. +if test "`$am_make -s -f confmf 2> /dev/null | grep -v 'ing directory'`" = "done"; then + am__include=include + am__quote= + _am_result=GNU +fi +# Now try BSD make style include. +if test "$am__include" = "#"; then + echo '.include "confinc"' > confmf + if test "`$am_make -s -f confmf 2> /dev/null`" = "done"; then + am__include=.include + am__quote="\"" + _am_result=BSD + fi +fi +AC_SUBST([am__include]) +AC_SUBST([am__quote]) +AC_MSG_RESULT([$_am_result]) +rm -f confinc confmf +]) + +# Fake the existence of programs that GNU maintainers use. -*- Autoconf -*- + +# Copyright (C) 1997, 1999, 2000, 2001, 2003, 2005 +# Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 4 + +# AM_MISSING_PROG(NAME, PROGRAM) +# ------------------------------ +AC_DEFUN([AM_MISSING_PROG], +[AC_REQUIRE([AM_MISSING_HAS_RUN]) +$1=${$1-"${am_missing_run}$2"} +AC_SUBST($1)]) + + +# AM_MISSING_HAS_RUN +# ------------------ +# Define MISSING if not defined so far and test if it supports --run. +# If it does, set am_missing_run to use it, otherwise, to nothing. +AC_DEFUN([AM_MISSING_HAS_RUN], +[AC_REQUIRE([AM_AUX_DIR_EXPAND])dnl +test x"${MISSING+set}" = xset || MISSING="\${SHELL} $am_aux_dir/missing" +# Use eval to expand $SHELL +if eval "$MISSING --run true"; then + am_missing_run="$MISSING --run " +else + am_missing_run= + AC_MSG_WARN([`missing' script is too old or missing]) +fi +]) + +# Copyright (C) 2003, 2004, 2005 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# AM_PROG_MKDIR_P +# --------------- +# Check whether `mkdir -p' is supported, fallback to mkinstalldirs otherwise. +# +# Automake 1.8 used `mkdir -m 0755 -p --' to ensure that directories +# created by `make install' are always world readable, even if the +# installer happens to have an overly restrictive umask (e.g. 077). +# This was a mistake. There are at least two reasons why we must not +# use `-m 0755': +# - it causes special bits like SGID to be ignored, +# - it may be too restrictive (some setups expect 775 directories). +# +# Do not use -m 0755 and let people choose whatever they expect by +# setting umask. +# +# We cannot accept any implementation of `mkdir' that recognizes `-p'. +# Some implementations (such as Solaris 8's) are not thread-safe: if a +# parallel make tries to run `mkdir -p a/b' and `mkdir -p a/c' +# concurrently, both version can detect that a/ is missing, but only +# one can create it and the other will error out. Consequently we +# restrict ourselves to GNU make (using the --version option ensures +# this.) +AC_DEFUN([AM_PROG_MKDIR_P], +[if mkdir -p --version . >/dev/null 2>&1 && test ! -d ./--version; then + # We used to keeping the `.' as first argument, in order to + # allow $(mkdir_p) to be used without argument. As in + # $(mkdir_p) $(somedir) + # where $(somedir) is conditionally defined. However this is wrong + # for two reasons: + # 1. if the package is installed by a user who cannot write `.' + # make install will fail, + # 2. the above comment should most certainly read + # $(mkdir_p) $(DESTDIR)$(somedir) + # so it does not work when $(somedir) is undefined and + # $(DESTDIR) is not. + # To support the latter case, we have to write + # test -z "$(somedir)" || $(mkdir_p) $(DESTDIR)$(somedir), + # so the `.' trick is pointless. + mkdir_p='mkdir -p --' +else + # On NextStep and OpenStep, the `mkdir' command does not + # recognize any option. It will interpret all options as + # directories to create, and then abort because `.' already + # exists. + for d in ./-p ./--version; + do + test -d $d && rmdir $d + done + # $(mkinstalldirs) is defined by Automake if mkinstalldirs exists. + if test -f "$ac_aux_dir/mkinstalldirs"; then + mkdir_p='$(mkinstalldirs)' + else + mkdir_p='$(install_sh) -d' + fi +fi +AC_SUBST([mkdir_p])]) + +# Helper functions for option handling. -*- Autoconf -*- + +# Copyright (C) 2001, 2002, 2003, 2005 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 3 + +# _AM_MANGLE_OPTION(NAME) +# ----------------------- +AC_DEFUN([_AM_MANGLE_OPTION], +[[_AM_OPTION_]m4_bpatsubst($1, [[^a-zA-Z0-9_]], [_])]) + +# _AM_SET_OPTION(NAME) +# ------------------------------ +# Set option NAME. Presently that only means defining a flag for this option. +AC_DEFUN([_AM_SET_OPTION], +[m4_define(_AM_MANGLE_OPTION([$1]), 1)]) + +# _AM_SET_OPTIONS(OPTIONS) +# ---------------------------------- +# OPTIONS is a space-separated list of Automake options. +AC_DEFUN([_AM_SET_OPTIONS], +[AC_FOREACH([_AM_Option], [$1], [_AM_SET_OPTION(_AM_Option)])]) + +# _AM_IF_OPTION(OPTION, IF-SET, [IF-NOT-SET]) +# ------------------------------------------- +# Execute IF-SET if OPTION is set, IF-NOT-SET otherwise. +AC_DEFUN([_AM_IF_OPTION], +[m4_ifset(_AM_MANGLE_OPTION([$1]), [$2], [$3])]) + +# Check to make sure that the build environment is sane. -*- Autoconf -*- + +# Copyright (C) 1996, 1997, 2000, 2001, 2003, 2005 +# Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 4 + +# AM_SANITY_CHECK +# --------------- +AC_DEFUN([AM_SANITY_CHECK], +[AC_MSG_CHECKING([whether build environment is sane]) +# Just in case +sleep 1 +echo timestamp > conftest.file +# Do `set' in a subshell so we don't clobber the current shell's +# arguments. Must try -L first in case configure is actually a +# symlink; some systems play weird games with the mod time of symlinks +# (eg FreeBSD returns the mod time of the symlink's containing +# directory). +if ( + set X `ls -Lt $srcdir/configure conftest.file 2> /dev/null` + if test "$[*]" = "X"; then + # -L didn't work. + set X `ls -t $srcdir/configure conftest.file` + fi + rm -f conftest.file + if test "$[*]" != "X $srcdir/configure conftest.file" \ + && test "$[*]" != "X conftest.file $srcdir/configure"; then + + # If neither matched, then we have a broken ls. This can happen + # if, for instance, CONFIG_SHELL is bash and it inherits a + # broken ls alias from the environment. This has actually + # happened. Such a system could not be considered "sane". + AC_MSG_ERROR([ls -t appears to fail. Make sure there is not a broken +alias in your environment]) + fi + + test "$[2]" = conftest.file + ) +then + # Ok. + : +else + AC_MSG_ERROR([newly created file is older than distributed files! +Check your system clock]) +fi +AC_MSG_RESULT(yes)]) + +# Copyright (C) 2001, 2003, 2005 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# AM_PROG_INSTALL_STRIP +# --------------------- +# One issue with vendor `install' (even GNU) is that you can't +# specify the program used to strip binaries. This is especially +# annoying in cross-compiling environments, where the build's strip +# is unlikely to handle the host's binaries. +# Fortunately install-sh will honor a STRIPPROG variable, so we +# always use install-sh in `make install-strip', and initialize +# STRIPPROG with the value of the STRIP variable (set by the user). +AC_DEFUN([AM_PROG_INSTALL_STRIP], +[AC_REQUIRE([AM_PROG_INSTALL_SH])dnl +# Installed binaries are usually stripped using `strip' when the user +# run `make install-strip'. However `strip' might not be the right +# tool to use in cross-compilation environments, therefore Automake +# will honor the `STRIP' environment variable to overrule this program. +dnl Don't test for $cross_compiling = yes, because it might be `maybe'. +if test "$cross_compiling" != no; then + AC_CHECK_TOOL([STRIP], [strip], :) +fi +INSTALL_STRIP_PROGRAM="\${SHELL} \$(install_sh) -c -s" +AC_SUBST([INSTALL_STRIP_PROGRAM])]) + +# Check how to create a tarball. -*- Autoconf -*- + +# Copyright (C) 2004, 2005 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# serial 2 + +# _AM_PROG_TAR(FORMAT) +# -------------------- +# Check how to create a tarball in format FORMAT. +# FORMAT should be one of `v7', `ustar', or `pax'. +# +# Substitute a variable $(am__tar) that is a command +# writing to stdout a FORMAT-tarball containing the directory +# $tardir. +# tardir=directory && $(am__tar) > result.tar +# +# Substitute a variable $(am__untar) that extract such +# a tarball read from stdin. +# $(am__untar) < result.tar +AC_DEFUN([_AM_PROG_TAR], +[# Always define AMTAR for backward compatibility. +AM_MISSING_PROG([AMTAR], [tar]) +m4_if([$1], [v7], + [am__tar='${AMTAR} chof - "$$tardir"'; am__untar='${AMTAR} xf -'], + [m4_case([$1], [ustar],, [pax],, + [m4_fatal([Unknown tar format])]) +AC_MSG_CHECKING([how to create a $1 tar archive]) +# Loop over all known methods to create a tar archive until one works. +_am_tools='gnutar m4_if([$1], [ustar], [plaintar]) pax cpio none' +_am_tools=${am_cv_prog_tar_$1-$_am_tools} +# Do not fold the above two line into one, because Tru64 sh and +# Solaris sh will not grok spaces in the rhs of `-'. +for _am_tool in $_am_tools +do + case $_am_tool in + gnutar) + for _am_tar in tar gnutar gtar; + do + AM_RUN_LOG([$_am_tar --version]) && break + done + am__tar="$_am_tar --format=m4_if([$1], [pax], [posix], [$1]) -chf - "'"$$tardir"' + am__tar_="$_am_tar --format=m4_if([$1], [pax], [posix], [$1]) -chf - "'"$tardir"' + am__untar="$_am_tar -xf -" + ;; + plaintar) + # Must skip GNU tar: if it does not support --format= it doesn't create + # ustar tarball either. + (tar --version) >/dev/null 2>&1 && continue + am__tar='tar chf - "$$tardir"' + am__tar_='tar chf - "$tardir"' + am__untar='tar xf -' + ;; + pax) + am__tar='pax -L -x $1 -w "$$tardir"' + am__tar_='pax -L -x $1 -w "$tardir"' + am__untar='pax -r' + ;; + cpio) + am__tar='find "$$tardir" -print | cpio -o -H $1 -L' + am__tar_='find "$tardir" -print | cpio -o -H $1 -L' + am__untar='cpio -i -H $1 -d' + ;; + none) + am__tar=false + am__tar_=false + am__untar=false + ;; + esac + + # If the value was cached, stop now. We just wanted to have am__tar + # and am__untar set. + test -n "${am_cv_prog_tar_$1}" && break + + # tar/untar a dummy directory, and stop if the command works + rm -rf conftest.dir + mkdir conftest.dir + echo GrepMe > conftest.dir/file + AM_RUN_LOG([tardir=conftest.dir && eval $am__tar_ >conftest.tar]) + rm -rf conftest.dir + if test -s conftest.tar; then + AM_RUN_LOG([$am__untar /dev/null 2>&1 && break + fi +done +rm -rf conftest.dir + +AC_CACHE_VAL([am_cv_prog_tar_$1], [am_cv_prog_tar_$1=$_am_tool]) +AC_MSG_RESULT([$am_cv_prog_tar_$1])]) +AC_SUBST([am__tar]) +AC_SUBST([am__untar]) +]) # _AM_PROG_TAR + +m4_include([m4/ax_boost_base.m4]) diff --git a/mosesdecoder/contrib/memscore/config.h.in b/mosesdecoder/contrib/memscore/config.h.in new file mode 100644 index 0000000000000000000000000000000000000000..05b54369cffeb8031b7b2652b2c023bbafe714a6 --- /dev/null +++ b/mosesdecoder/contrib/memscore/config.h.in @@ -0,0 +1,79 @@ +/* config.h.in. Generated from configure.ac by autoheader. */ + +/* Define to enable channel scorer */ +#undef ENABLE_CHANNEL_SCORER + +/* define if the Boost library is available */ +#undef HAVE_BOOST + +/* Define to 1 if you have the header file. */ +#undef HAVE_INTTYPES_H + +/* flag for IRSTLM */ +#undef HAVE_IRSTLM + +/* Define to 1 if you have the `gsl' library (-lgsl). */ +#undef HAVE_LIBGSL + +/* Define to 1 if you have the `gslcblas' library (-lgslcblas). */ +#undef HAVE_LIBGSLCBLAS + +/* Define to 1 if you have the `m' library (-lm). */ +#undef HAVE_LIBM + +/* Define to 1 if you have the `z' library (-lz). */ +#undef HAVE_LIBZ + +/* Define to 1 if you have the header file. */ +#undef HAVE_MEMORY_H + +/* Define to 1 if the system has the type `ptrdiff_t'. */ +#undef HAVE_PTRDIFF_T + +/* Define to 1 if you have the header file. */ +#undef HAVE_STDINT_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_STDLIB_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_STRINGS_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_STRING_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_STAT_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_TYPES_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_UNISTD_H + +/* Name of package */ +#undef PACKAGE + +/* Define to the address where bug reports for this package should be sent. */ +#undef PACKAGE_BUGREPORT + +/* Define to the full name of this package. */ +#undef PACKAGE_NAME + +/* Define to the full name and version of this package. */ +#undef PACKAGE_STRING + +/* Define to the one symbol short name of this package. */ +#undef PACKAGE_TARNAME + +/* Define to the version of this package. */ +#undef PACKAGE_VERSION + +/* Define to 1 if you have the ANSI C header files. */ +#undef STDC_HEADERS + +/* Version number of package */ +#undef VERSION + +/* Define to `unsigned int' if does not define. */ +#undef size_t diff --git a/mosesdecoder/contrib/memscore/datastorage.h b/mosesdecoder/contrib/memscore/datastorage.h new file mode 100644 index 0000000000000000000000000000000000000000..0eb4cded1b8d23f3a7ceed6893b46d1728b3b712 --- /dev/null +++ b/mosesdecoder/contrib/memscore/datastorage.h @@ -0,0 +1,52 @@ +// memscore - in-memory phrase scoring for Statistical Machine Translation +// Christian Hardmeier, FBK-irst, Trento, 2010 +// $Id$ + +#ifndef DATASTORAGE_H +#define DATASTORAGE_H + +#include "memscore.h" + +template +class DataStorage +{ +private: + T *base_; + size_t pos_; + size_t chunk_size_; + + DataStorage() : base_(NULL), pos_(0) { + int ps = getpagesize(); + chunk_size_ = 10 * ps * sizeof(T); + base_ = new T[chunk_size_]; + } + + DataStorage(const DataStorage &cc) { + abort(); + } + + ~DataStorage() {} + +public: + static DataStorage &get_instance() { + static DataStorage instance; + return instance; + } + + T *alloc(size_t count) { + if(count == 0) + return NULL; + + // The memory leak is intended. + if(pos_ + count > chunk_size_) { + base_ = new T[chunk_size_]; + pos_ = 0; + } + + T *ret = base_ + pos_; + pos_ += count; + return ret; + } +}; + +#endif diff --git a/mosesdecoder/contrib/memscore/depcomp b/mosesdecoder/contrib/memscore/depcomp new file mode 100644 index 0000000000000000000000000000000000000000..04701da536f33a7c39d7bb01b87a70ae3a776df5 --- /dev/null +++ b/mosesdecoder/contrib/memscore/depcomp @@ -0,0 +1,530 @@ +#! /bin/sh +# depcomp - compile a program generating dependencies as side-effects + +scriptversion=2005-07-09.11 + +# Copyright (C) 1999, 2000, 2003, 2004, 2005 Free Software Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. + +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + +# Originally written by Alexandre Oliva . + +case $1 in + '') + echo "$0: No command. Try \`$0 --help' for more information." 1>&2 + exit 1; + ;; + -h | --h*) + cat <<\EOF +Usage: depcomp [--help] [--version] PROGRAM [ARGS] + +Run PROGRAMS ARGS to compile a file, generating dependencies +as side-effects. + +Environment variables: + depmode Dependency tracking mode. + source Source file read by `PROGRAMS ARGS'. + object Object file output by `PROGRAMS ARGS'. + DEPDIR directory where to store dependencies. + depfile Dependency file to output. + tmpdepfile Temporary file to use when outputing dependencies. + libtool Whether libtool is used (yes/no). + +Report bugs to . +EOF + exit $? + ;; + -v | --v*) + echo "depcomp $scriptversion" + exit $? + ;; +esac + +if test -z "$depmode" || test -z "$source" || test -z "$object"; then + echo "depcomp: Variables source, object and depmode must be set" 1>&2 + exit 1 +fi + +# Dependencies for sub/bar.o or sub/bar.obj go into sub/.deps/bar.Po. +depfile=${depfile-`echo "$object" | + sed 's|[^\\/]*$|'${DEPDIR-.deps}'/&|;s|\.\([^.]*\)$|.P\1|;s|Pobj$|Po|'`} +tmpdepfile=${tmpdepfile-`echo "$depfile" | sed 's/\.\([^.]*\)$/.T\1/'`} + +rm -f "$tmpdepfile" + +# Some modes work just like other modes, but use different flags. We +# parameterize here, but still list the modes in the big case below, +# to make depend.m4 easier to write. Note that we *cannot* use a case +# here, because this file can only contain one case statement. +if test "$depmode" = hp; then + # HP compiler uses -M and no extra arg. + gccflag=-M + depmode=gcc +fi + +if test "$depmode" = dashXmstdout; then + # This is just like dashmstdout with a different argument. + dashmflag=-xM + depmode=dashmstdout +fi + +case "$depmode" in +gcc3) +## gcc 3 implements dependency tracking that does exactly what +## we want. Yay! Note: for some reason libtool 1.4 doesn't like +## it if -MD -MP comes after the -MF stuff. Hmm. + "$@" -MT "$object" -MD -MP -MF "$tmpdepfile" + stat=$? + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile" + exit $stat + fi + mv "$tmpdepfile" "$depfile" + ;; + +gcc) +## There are various ways to get dependency output from gcc. Here's +## why we pick this rather obscure method: +## - Don't want to use -MD because we'd like the dependencies to end +## up in a subdir. Having to rename by hand is ugly. +## (We might end up doing this anyway to support other compilers.) +## - The DEPENDENCIES_OUTPUT environment variable makes gcc act like +## -MM, not -M (despite what the docs say). +## - Using -M directly means running the compiler twice (even worse +## than renaming). + if test -z "$gccflag"; then + gccflag=-MD, + fi + "$@" -Wp,"$gccflag$tmpdepfile" + stat=$? + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile" + exit $stat + fi + rm -f "$depfile" + echo "$object : \\" > "$depfile" + alpha=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz +## The second -e expression handles DOS-style file names with drive letters. + sed -e 's/^[^:]*: / /' \ + -e 's/^['$alpha']:\/[^:]*: / /' < "$tmpdepfile" >> "$depfile" +## This next piece of magic avoids the `deleted header file' problem. +## The problem is that when a header file which appears in a .P file +## is deleted, the dependency causes make to die (because there is +## typically no way to rebuild the header). We avoid this by adding +## dummy dependencies for each header file. Too bad gcc doesn't do +## this for us directly. + tr ' ' ' +' < "$tmpdepfile" | +## Some versions of gcc put a space before the `:'. On the theory +## that the space means something, we add a space to the output as +## well. +## Some versions of the HPUX 10.20 sed can't process this invocation +## correctly. Breaking it into two sed invocations is a workaround. + sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile" + rm -f "$tmpdepfile" + ;; + +hp) + # This case exists only to let depend.m4 do its work. It works by + # looking at the text of this script. This case will never be run, + # since it is checked for above. + exit 1 + ;; + +sgi) + if test "$libtool" = yes; then + "$@" "-Wp,-MDupdate,$tmpdepfile" + else + "$@" -MDupdate "$tmpdepfile" + fi + stat=$? + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile" + exit $stat + fi + rm -f "$depfile" + + if test -f "$tmpdepfile"; then # yes, the sourcefile depend on other files + echo "$object : \\" > "$depfile" + + # Clip off the initial element (the dependent). Don't try to be + # clever and replace this with sed code, as IRIX sed won't handle + # lines with more than a fixed number of characters (4096 in + # IRIX 6.2 sed, 8192 in IRIX 6.5). We also remove comment lines; + # the IRIX cc adds comments like `#:fec' to the end of the + # dependency line. + tr ' ' ' +' < "$tmpdepfile" \ + | sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' | \ + tr ' +' ' ' >> $depfile + echo >> $depfile + + # The second pass generates a dummy entry for each header file. + tr ' ' ' +' < "$tmpdepfile" \ + | sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' -e 's/$/:/' \ + >> $depfile + else + # The sourcefile does not contain any dependencies, so just + # store a dummy comment line, to avoid errors with the Makefile + # "include basename.Plo" scheme. + echo "#dummy" > "$depfile" + fi + rm -f "$tmpdepfile" + ;; + +aix) + # The C for AIX Compiler uses -M and outputs the dependencies + # in a .u file. In older versions, this file always lives in the + # current directory. Also, the AIX compiler puts `$object:' at the + # start of each line; $object doesn't have directory information. + # Version 6 uses the directory in both cases. + stripped=`echo "$object" | sed 's/\(.*\)\..*$/\1/'` + tmpdepfile="$stripped.u" + if test "$libtool" = yes; then + "$@" -Wc,-M + else + "$@" -M + fi + stat=$? + + if test -f "$tmpdepfile"; then : + else + stripped=`echo "$stripped" | sed 's,^.*/,,'` + tmpdepfile="$stripped.u" + fi + + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile" + exit $stat + fi + + if test -f "$tmpdepfile"; then + outname="$stripped.o" + # Each line is of the form `foo.o: dependent.h'. + # Do two passes, one to just change these to + # `$object: dependent.h' and one to simply `dependent.h:'. + sed -e "s,^$outname:,$object :," < "$tmpdepfile" > "$depfile" + sed -e "s,^$outname: \(.*\)$,\1:," < "$tmpdepfile" >> "$depfile" + else + # The sourcefile does not contain any dependencies, so just + # store a dummy comment line, to avoid errors with the Makefile + # "include basename.Plo" scheme. + echo "#dummy" > "$depfile" + fi + rm -f "$tmpdepfile" + ;; + +icc) + # Intel's C compiler understands `-MD -MF file'. However on + # icc -MD -MF foo.d -c -o sub/foo.o sub/foo.c + # ICC 7.0 will fill foo.d with something like + # foo.o: sub/foo.c + # foo.o: sub/foo.h + # which is wrong. We want: + # sub/foo.o: sub/foo.c + # sub/foo.o: sub/foo.h + # sub/foo.c: + # sub/foo.h: + # ICC 7.1 will output + # foo.o: sub/foo.c sub/foo.h + # and will wrap long lines using \ : + # foo.o: sub/foo.c ... \ + # sub/foo.h ... \ + # ... + + "$@" -MD -MF "$tmpdepfile" + stat=$? + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile" + exit $stat + fi + rm -f "$depfile" + # Each line is of the form `foo.o: dependent.h', + # or `foo.o: dep1.h dep2.h \', or ` dep3.h dep4.h \'. + # Do two passes, one to just change these to + # `$object: dependent.h' and one to simply `dependent.h:'. + sed "s,^[^:]*:,$object :," < "$tmpdepfile" > "$depfile" + # Some versions of the HPUX 10.20 sed can't process this invocation + # correctly. Breaking it into two sed invocations is a workaround. + sed 's,^[^:]*: \(.*\)$,\1,;s/^\\$//;/^$/d;/:$/d' < "$tmpdepfile" | + sed -e 's/$/ :/' >> "$depfile" + rm -f "$tmpdepfile" + ;; + +tru64) + # The Tru64 compiler uses -MD to generate dependencies as a side + # effect. `cc -MD -o foo.o ...' puts the dependencies into `foo.o.d'. + # At least on Alpha/Redhat 6.1, Compaq CCC V6.2-504 seems to put + # dependencies in `foo.d' instead, so we check for that too. + # Subdirectories are respected. + dir=`echo "$object" | sed -e 's|/[^/]*$|/|'` + test "x$dir" = "x$object" && dir= + base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'` + + if test "$libtool" = yes; then + # With Tru64 cc, shared objects can also be used to make a + # static library. This mecanism is used in libtool 1.4 series to + # handle both shared and static libraries in a single compilation. + # With libtool 1.4, dependencies were output in $dir.libs/$base.lo.d. + # + # With libtool 1.5 this exception was removed, and libtool now + # generates 2 separate objects for the 2 libraries. These two + # compilations output dependencies in in $dir.libs/$base.o.d and + # in $dir$base.o.d. We have to check for both files, because + # one of the two compilations can be disabled. We should prefer + # $dir$base.o.d over $dir.libs/$base.o.d because the latter is + # automatically cleaned when .libs/ is deleted, while ignoring + # the former would cause a distcleancheck panic. + tmpdepfile1=$dir.libs/$base.lo.d # libtool 1.4 + tmpdepfile2=$dir$base.o.d # libtool 1.5 + tmpdepfile3=$dir.libs/$base.o.d # libtool 1.5 + tmpdepfile4=$dir.libs/$base.d # Compaq CCC V6.2-504 + "$@" -Wc,-MD + else + tmpdepfile1=$dir$base.o.d + tmpdepfile2=$dir$base.d + tmpdepfile3=$dir$base.d + tmpdepfile4=$dir$base.d + "$@" -MD + fi + + stat=$? + if test $stat -eq 0; then : + else + rm -f "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" "$tmpdepfile4" + exit $stat + fi + + for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" "$tmpdepfile4" + do + test -f "$tmpdepfile" && break + done + if test -f "$tmpdepfile"; then + sed -e "s,^.*\.[a-z]*:,$object:," < "$tmpdepfile" > "$depfile" + # That's a tab and a space in the []. + sed -e 's,^.*\.[a-z]*:[ ]*,,' -e 's,$,:,' < "$tmpdepfile" >> "$depfile" + else + echo "#dummy" > "$depfile" + fi + rm -f "$tmpdepfile" + ;; + +#nosideeffect) + # This comment above is used by automake to tell side-effect + # dependency tracking mechanisms from slower ones. + +dashmstdout) + # Important note: in order to support this mode, a compiler *must* + # always write the preprocessed file to stdout, regardless of -o. + "$@" || exit $? + + # Remove the call to Libtool. + if test "$libtool" = yes; then + while test $1 != '--mode=compile'; do + shift + done + shift + fi + + # Remove `-o $object'. + IFS=" " + for arg + do + case $arg in + -o) + shift + ;; + $object) + shift + ;; + *) + set fnord "$@" "$arg" + shift # fnord + shift # $arg + ;; + esac + done + + test -z "$dashmflag" && dashmflag=-M + # Require at least two characters before searching for `:' + # in the target name. This is to cope with DOS-style filenames: + # a dependency such as `c:/foo/bar' could be seen as target `c' otherwise. + "$@" $dashmflag | + sed 's:^[ ]*[^: ][^:][^:]*\:[ ]*:'"$object"'\: :' > "$tmpdepfile" + rm -f "$depfile" + cat < "$tmpdepfile" > "$depfile" + tr ' ' ' +' < "$tmpdepfile" | \ +## Some versions of the HPUX 10.20 sed can't process this invocation +## correctly. Breaking it into two sed invocations is a workaround. + sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile" + rm -f "$tmpdepfile" + ;; + +dashXmstdout) + # This case only exists to satisfy depend.m4. It is never actually + # run, as this mode is specially recognized in the preamble. + exit 1 + ;; + +makedepend) + "$@" || exit $? + # Remove any Libtool call + if test "$libtool" = yes; then + while test $1 != '--mode=compile'; do + shift + done + shift + fi + # X makedepend + shift + cleared=no + for arg in "$@"; do + case $cleared in + no) + set ""; shift + cleared=yes ;; + esac + case "$arg" in + -D*|-I*) + set fnord "$@" "$arg"; shift ;; + # Strip any option that makedepend may not understand. Remove + # the object too, otherwise makedepend will parse it as a source file. + -*|$object) + ;; + *) + set fnord "$@" "$arg"; shift ;; + esac + done + obj_suffix="`echo $object | sed 's/^.*\././'`" + touch "$tmpdepfile" + ${MAKEDEPEND-makedepend} -o"$obj_suffix" -f"$tmpdepfile" "$@" + rm -f "$depfile" + cat < "$tmpdepfile" > "$depfile" + sed '1,2d' "$tmpdepfile" | tr ' ' ' +' | \ +## Some versions of the HPUX 10.20 sed can't process this invocation +## correctly. Breaking it into two sed invocations is a workaround. + sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile" + rm -f "$tmpdepfile" "$tmpdepfile".bak + ;; + +cpp) + # Important note: in order to support this mode, a compiler *must* + # always write the preprocessed file to stdout. + "$@" || exit $? + + # Remove the call to Libtool. + if test "$libtool" = yes; then + while test $1 != '--mode=compile'; do + shift + done + shift + fi + + # Remove `-o $object'. + IFS=" " + for arg + do + case $arg in + -o) + shift + ;; + $object) + shift + ;; + *) + set fnord "$@" "$arg" + shift # fnord + shift # $arg + ;; + esac + done + + "$@" -E | + sed -n -e '/^# [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' \ + -e '/^#line [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' | + sed '$ s: \\$::' > "$tmpdepfile" + rm -f "$depfile" + echo "$object : \\" > "$depfile" + cat < "$tmpdepfile" >> "$depfile" + sed < "$tmpdepfile" '/^$/d;s/^ //;s/ \\$//;s/$/ :/' >> "$depfile" + rm -f "$tmpdepfile" + ;; + +msvisualcpp) + # Important note: in order to support this mode, a compiler *must* + # always write the preprocessed file to stdout, regardless of -o, + # because we must use -o when running libtool. + "$@" || exit $? + IFS=" " + for arg + do + case "$arg" in + "-Gm"|"/Gm"|"-Gi"|"/Gi"|"-ZI"|"/ZI") + set fnord "$@" + shift + shift + ;; + *) + set fnord "$@" "$arg" + shift + shift + ;; + esac + done + "$@" -E | + sed -n '/^#line [0-9][0-9]* "\([^"]*\)"/ s::echo "`cygpath -u \\"\1\\"`":p' | sort | uniq > "$tmpdepfile" + rm -f "$depfile" + echo "$object : \\" > "$depfile" + . "$tmpdepfile" | sed 's% %\\ %g' | sed -n '/^\(.*\)$/ s:: \1 \\:p' >> "$depfile" + echo " " >> "$depfile" + . "$tmpdepfile" | sed 's% %\\ %g' | sed -n '/^\(.*\)$/ s::\1\::p' >> "$depfile" + rm -f "$tmpdepfile" + ;; + +none) + exec "$@" + ;; + +*) + echo "Unknown depmode $depmode" 1>&2 + exit 1 + ;; +esac + +exit 0 + +# Local Variables: +# mode: shell-script +# sh-indentation: 2 +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-end: "$" +# End: diff --git a/mosesdecoder/contrib/memscore/install-sh b/mosesdecoder/contrib/memscore/install-sh new file mode 100644 index 0000000000000000000000000000000000000000..4d4a9519eaf88b18fb157dfe5fae59c1c5d005c7 --- /dev/null +++ b/mosesdecoder/contrib/memscore/install-sh @@ -0,0 +1,323 @@ +#!/bin/sh +# install - install a program, script, or datafile + +scriptversion=2005-05-14.22 + +# This originates from X11R5 (mit/util/scripts/install.sh), which was +# later released in X11R6 (xc/config/util/install.sh) with the +# following copyright and license. +# +# Copyright (C) 1994 X Consortium +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN +# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNEC- +# TION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# Except as contained in this notice, the name of the X Consortium shall not +# be used in advertising or otherwise to promote the sale, use or other deal- +# ings in this Software without prior written authorization from the X Consor- +# tium. +# +# +# FSF changes to this file are in the public domain. +# +# Calling this script install-sh is preferred over install.sh, to prevent +# `make' implicit rules from creating a file called install from it +# when there is no Makefile. +# +# This script is compatible with the BSD install script, but was written +# from scratch. It can only install one file at a time, a restriction +# shared with many OS's install programs. + +# set DOITPROG to echo to test this script + +# Don't use :- since 4.3BSD and earlier shells don't like it. +doit="${DOITPROG-}" + +# put in absolute paths if you don't have them in your path; or use env. vars. + +mvprog="${MVPROG-mv}" +cpprog="${CPPROG-cp}" +chmodprog="${CHMODPROG-chmod}" +chownprog="${CHOWNPROG-chown}" +chgrpprog="${CHGRPPROG-chgrp}" +stripprog="${STRIPPROG-strip}" +rmprog="${RMPROG-rm}" +mkdirprog="${MKDIRPROG-mkdir}" + +chmodcmd="$chmodprog 0755" +chowncmd= +chgrpcmd= +stripcmd= +rmcmd="$rmprog -f" +mvcmd="$mvprog" +src= +dst= +dir_arg= +dstarg= +no_target_directory= + +usage="Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE + or: $0 [OPTION]... SRCFILES... DIRECTORY + or: $0 [OPTION]... -t DIRECTORY SRCFILES... + or: $0 [OPTION]... -d DIRECTORIES... + +In the 1st form, copy SRCFILE to DSTFILE. +In the 2nd and 3rd, copy all SRCFILES to DIRECTORY. +In the 4th, create DIRECTORIES. + +Options: +-c (ignored) +-d create directories instead of installing files. +-g GROUP $chgrpprog installed files to GROUP. +-m MODE $chmodprog installed files to MODE. +-o USER $chownprog installed files to USER. +-s $stripprog installed files. +-t DIRECTORY install into DIRECTORY. +-T report an error if DSTFILE is a directory. +--help display this help and exit. +--version display version info and exit. + +Environment variables override the default commands: + CHGRPPROG CHMODPROG CHOWNPROG CPPROG MKDIRPROG MVPROG RMPROG STRIPPROG +" + +while test -n "$1"; do + case $1 in + -c) shift + continue;; + + -d) dir_arg=true + shift + continue;; + + -g) chgrpcmd="$chgrpprog $2" + shift + shift + continue;; + + --help) echo "$usage"; exit $?;; + + -m) chmodcmd="$chmodprog $2" + shift + shift + continue;; + + -o) chowncmd="$chownprog $2" + shift + shift + continue;; + + -s) stripcmd=$stripprog + shift + continue;; + + -t) dstarg=$2 + shift + shift + continue;; + + -T) no_target_directory=true + shift + continue;; + + --version) echo "$0 $scriptversion"; exit $?;; + + *) # When -d is used, all remaining arguments are directories to create. + # When -t is used, the destination is already specified. + test -n "$dir_arg$dstarg" && break + # Otherwise, the last argument is the destination. Remove it from $@. + for arg + do + if test -n "$dstarg"; then + # $@ is not empty: it contains at least $arg. + set fnord "$@" "$dstarg" + shift # fnord + fi + shift # arg + dstarg=$arg + done + break;; + esac +done + +if test -z "$1"; then + if test -z "$dir_arg"; then + echo "$0: no input file specified." >&2 + exit 1 + fi + # It's OK to call `install-sh -d' without argument. + # This can happen when creating conditional directories. + exit 0 +fi + +for src +do + # Protect names starting with `-'. + case $src in + -*) src=./$src ;; + esac + + if test -n "$dir_arg"; then + dst=$src + src= + + if test -d "$dst"; then + mkdircmd=: + chmodcmd= + else + mkdircmd=$mkdirprog + fi + else + # Waiting for this to be detected by the "$cpprog $src $dsttmp" command + # might cause directories to be created, which would be especially bad + # if $src (and thus $dsttmp) contains '*'. + if test ! -f "$src" && test ! -d "$src"; then + echo "$0: $src does not exist." >&2 + exit 1 + fi + + if test -z "$dstarg"; then + echo "$0: no destination specified." >&2 + exit 1 + fi + + dst=$dstarg + # Protect names starting with `-'. + case $dst in + -*) dst=./$dst ;; + esac + + # If destination is a directory, append the input filename; won't work + # if double slashes aren't ignored. + if test -d "$dst"; then + if test -n "$no_target_directory"; then + echo "$0: $dstarg: Is a directory" >&2 + exit 1 + fi + dst=$dst/`basename "$src"` + fi + fi + + # This sed command emulates the dirname command. + dstdir=`echo "$dst" | sed -e 's,/*$,,;s,[^/]*$,,;s,/*$,,;s,^$,.,'` + + # Make sure that the destination directory exists. + + # Skip lots of stat calls in the usual case. + if test ! -d "$dstdir"; then + defaultIFS=' + ' + IFS="${IFS-$defaultIFS}" + + oIFS=$IFS + # Some sh's can't handle IFS=/ for some reason. + IFS='%' + set x `echo "$dstdir" | sed -e 's@/@%@g' -e 's@^%@/@'` + shift + IFS=$oIFS + + pathcomp= + + while test $# -ne 0 ; do + pathcomp=$pathcomp$1 + shift + if test ! -d "$pathcomp"; then + $mkdirprog "$pathcomp" + # mkdir can fail with a `File exist' error in case several + # install-sh are creating the directory concurrently. This + # is OK. + test -d "$pathcomp" || exit + fi + pathcomp=$pathcomp/ + done + fi + + if test -n "$dir_arg"; then + $doit $mkdircmd "$dst" \ + && { test -z "$chowncmd" || $doit $chowncmd "$dst"; } \ + && { test -z "$chgrpcmd" || $doit $chgrpcmd "$dst"; } \ + && { test -z "$stripcmd" || $doit $stripcmd "$dst"; } \ + && { test -z "$chmodcmd" || $doit $chmodcmd "$dst"; } + + else + dstfile=`basename "$dst"` + + # Make a couple of temp file names in the proper directory. + dsttmp=$dstdir/_inst.$$_ + rmtmp=$dstdir/_rm.$$_ + + # Trap to clean up those temp files at exit. + trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0 + trap '(exit $?); exit' 1 2 13 15 + + # Copy the file name to the temp name. + $doit $cpprog "$src" "$dsttmp" && + + # and set any options; do chmod last to preserve setuid bits. + # + # If any of these fail, we abort the whole thing. If we want to + # ignore errors from any of these, just make sure not to ignore + # errors from the above "$doit $cpprog $src $dsttmp" command. + # + { test -z "$chowncmd" || $doit $chowncmd "$dsttmp"; } \ + && { test -z "$chgrpcmd" || $doit $chgrpcmd "$dsttmp"; } \ + && { test -z "$stripcmd" || $doit $stripcmd "$dsttmp"; } \ + && { test -z "$chmodcmd" || $doit $chmodcmd "$dsttmp"; } && + + # Now rename the file to the real destination. + { $doit $mvcmd -f "$dsttmp" "$dstdir/$dstfile" 2>/dev/null \ + || { + # The rename failed, perhaps because mv can't rename something else + # to itself, or perhaps because mv is so ancient that it does not + # support -f. + + # Now remove or move aside any old file at destination location. + # We try this two ways since rm can't unlink itself on some + # systems and the destination file might be busy for other + # reasons. In this case, the final cleanup might fail but the new + # file should still install successfully. + { + if test -f "$dstdir/$dstfile"; then + $doit $rmcmd -f "$dstdir/$dstfile" 2>/dev/null \ + || $doit $mvcmd -f "$dstdir/$dstfile" "$rmtmp" 2>/dev/null \ + || { + echo "$0: cannot unlink or rename $dstdir/$dstfile" >&2 + (exit 1); exit 1 + } + else + : + fi + } && + + # Now rename the file to the real destination. + $doit $mvcmd "$dsttmp" "$dstdir/$dstfile" + } + } + fi || { (exit 1); exit 1; } +done + +# The final little trick to "correctly" pass the exit status to the exit trap. +{ + (exit 0); exit 0 +} + +# Local variables: +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-end: "$" +# End: diff --git a/mosesdecoder/contrib/memscore/lexdecom.cpp b/mosesdecoder/contrib/memscore/lexdecom.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b0a47f4409181a0e7178c3508b73900e90435c3d --- /dev/null +++ b/mosesdecoder/contrib/memscore/lexdecom.cpp @@ -0,0 +1,183 @@ +/* + * File: lexdecom.cpp + * Author: Felipe Sánchez-Martínez, Universitat d'Alacant + * + * Created on 2010/01/27 + */ + +#include "lexdecom.h" +#include "scorer-impl.h" + +#include +#include + +PhraseScorer* +LexicalDecompositionPhraseScorer::create_scorer(const char *argv[], int &argp, bool reverse, const PhraseScorerFactory &ptf) +{ + + if(argv[argp] == NULL) + usage(); + + String lwfile(argv[argp++]); + + return new LexicalDecompositionPhraseScorer(ptf.get_phrase_table(), reverse, lwfile, argv, argp, ptf); +} + +LexicalDecompositionPhraseScorer::LexicalDecompositionPhraseScorer(PhraseTable &pd, bool reverse, + const String &weightfile, const char *argv[], int &argp, + const PhraseScorerFactory &ptf) : + PhraseScorer(pd, reverse) +{ + + black_box_scorer = AbsoluteDiscountPhraseScorer::create_scorer(argv, argp, reverse, ptf); + + std::ifstream wfile(weightfile.c_str()); + + //Code copied from LexicalWeightPhraseScorer; it should be factored !! -- 2010/01/27 + + std::cerr<<"Reading lexical weights from '"<> src >> tgt >> weight; + Count src_id = PhraseText::index_word(src); + Count tgt_id = PhraseText::index_word(tgt); + weight_map_.insert(std::make_pair(std::make_pair(src_id, tgt_id), weight)); + } + + wfile.close(); + std::cerr<<"done."<second; +} + +void +LexicalDecompositionPhraseScorer::do_score_phrases() +{ + + //Estimate p(J|I) = p(src_len|tgt_len) + + black_box_scorer->score_phrases(); + + std::cerr<<"LexicalDecompositionPhraseScorer::do_score_phrases"< > count_srclen_tgtlen; + std::map total_tgtlen; + + for(PhraseTable::iterator it = phrase_table_.begin(); it != phrase_table_.end(); it++) { + const PhrasePairInfo &ppair = *it; + PhraseInfo &src = phrase_table_.get_src_phrase(ppair.get_src()); + PhraseInfo &tgt = phrase_table_.get_tgt_phrase(ppair.get_tgt()); + unsigned src_len = src.get_phrase().size(); + unsigned tgt_len = tgt.get_phrase().size(); + + /*//Debug code + for (unsigned i=0; i "< "< >::iterator its; + std::map::iterator itt; + + for (its=count_srclen_tgtlen.begin(); its!=count_srclen_tgtlen.end(); its++) { + unsigned src_len=its->first; + + for(itt=its->second.begin(); itt!=its->second.end(); itt++) { + unsigned tgt_len=itt->first; + Count cnt=itt->second; + prob_srclen_tgtlen_[src_len][tgt_len] = static_cast(cnt)/static_cast(total_tgtlen[tgt_len]); + } + } +} + +Score +LexicalDecompositionPhraseScorer::get_noisy_or_combination(Count src_word, PhraseInfo &tgt_phrase) +{ + + Score sc=1.0; + + unsigned tgt_len=tgt_phrase.get_phrase().size(); + + for(unsigned i=0; iget_src()); + PhraseInfo &tgt_phrase = phrase_table_.get_tgt_phrase(it->get_tgt()); + + unsigned src_len=src_phrase.get_phrase().size(); + unsigned tgt_len=tgt_phrase.get_phrase().size(); + + Score prod=1.0; + + for(unsigned j=0; j(black_box_scorer->get_discount()) * + tgt_phrase.get_distinct() / tgt_phrase.get_count(); + + Score ret_value = black_box_scorer->get_score(it) + (lambda * prod * prob_srclen_tgtlen_[src_len][tgt_len]); + + /* + //Debug code + for (unsigned i=0; i discount: "<get_discount()<<"; black box score: "<get_score(it) + <<"; lambda: "<get_count()<<"; score: "< +#include +#include +#include +#include + +#include +#include + +#include "phrasetable.h" +#include "phraselm.h" + +void PhraseLanguageModel::attach(PhraseInfoList &pilist) +{ + phrase_info_list_ = &pilist; + score_idx_ = pilist.register_data(1); +} + +void PhraseLanguageModel::compute_statistic() +{ + compute_lmscores(*phrase_info_list_, false); +} + +void PhraseLanguageModel::compute_lmscores(PhraseInfoList &phrase_info_list, bool closed_world) +{ + lmtable lm; + std::ifstream lmstream(lmfile_.c_str()); + lm.load(lmstream, lmfile_.c_str(), NULL, 0); + lm.setlogOOVpenalty(10000000); + + assert(!computation_done_); + + Score marginal_score = .0; + for(PhraseInfoList::iterator it = phrase_info_list.begin(); it != phrase_info_list.end(); ++it) { + PhraseInfo &pi = *it; + ngram ng(lm.getDict()); + Score lmscore = 0; + for(PhraseText::const_string_iterator it = pi.get_phrase().string_begin(); it != pi.get_phrase().string_end(); it++) { + ng.pushw(it->c_str()); + lmscore += lm.clprob(ng); + } + + pi.data(score_idx_) = exp10(lmscore); + marginal_score += pi.data(score_idx_); + } + + if(closed_world) + for(PhraseInfoList::iterator it = phrase_info_list.begin(); it != phrase_info_list.end(); ++it) { + PhraseInfo &pi = *it; + pi.data(score_idx_) /= marginal_score; + } + + computation_done_ = true; +} + +void ClosedPhraseLanguageModel::compute_statistic() +{ + compute_lmscores(*phrase_info_list_, true); +} diff --git a/mosesdecoder/contrib/memscore/phrasetable.h b/mosesdecoder/contrib/memscore/phrasetable.h new file mode 100644 index 0000000000000000000000000000000000000000..14d68d702686ff3790350d2c58ae2cb385e023a2 --- /dev/null +++ b/mosesdecoder/contrib/memscore/phrasetable.h @@ -0,0 +1,710 @@ +// memscore - in-memory phrase scoring for Statistical Machine Translation +// Christian Hardmeier, FBK-irst, Trento, 2010 +// $Id$ + +#ifndef PHRASETABLE_H +#define PHRASETABLE_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "datastorage.h" +#include "memscore.h" + +class PhraseText +{ + friend std::ostream &operator<<(std::ostream &os, const PhraseText &pt); + +private: + typedef boost::ptr_vector WordListType_; + typedef boost::bimap DictionaryType_; + + WordListType_ word_list_; + + static DictionaryType_ dictionary_; + static Count last_id_; + + typedef const String &(*LookupFunction_)(Count id); + +public: + typedef WordListType_::const_iterator const_iterator; + typedef boost::transform_iterator const_string_iterator; + typedef WordListType_::size_type size_type; + + PhraseText(const String &s); + + const_iterator begin() const { + return word_list_.begin(); + } + + const_iterator end() const { + return word_list_.end(); + } + + const_string_iterator string_begin() const { + return boost::make_transform_iterator(word_list_.begin(), dictionary_lookup); + } + + const_string_iterator string_end() const { + return boost::make_transform_iterator(word_list_.end(), dictionary_lookup); + } + + Count operator[](size_type i) const { + return word_list_[i]; + } + + const String &word(size_type i) const { + return dictionary_lookup(operator[](i)); + } + + size_type size() const { + return word_list_.size(); + } + + static const String &dictionary_lookup(Count id) { + DictionaryType_::right_const_iterator it = dictionary_.right.find(id); + assert(it != dictionary_.right.end()); + return it->second; + } + + static Count index_word(const String &word) { + Count id; + DictionaryType_::left_const_iterator it = dictionary_.left.find(word); + if(it != dictionary_.left.end()) + id = it->second; + else { + id = last_id_++; + dictionary_.insert(DictionaryType_::value_type(word, id)); + } + return id; + } +}; + +class PhraseInfo +{ + friend class boost::object_pool; + friend std::ostream &operator<<(std::ostream &os, const PhraseInfo &pt); + +protected: + Count data_size_; + + Count count_; + Count distinct_; + PhraseText phrase_; + Score *data_; + + Count n1_; + Count n2_; + Count n3plus_; + + PhraseInfo(Count data_size, const String &phrase) : + data_size_(data_size), count_(0), distinct_(0), phrase_(phrase), n1_(0), n2_(0), n3plus_(0) { + data_ = DataStorage::get_instance().alloc(data_size_); + } + +public: + Score &data(Count base, Count i = 0) { + assert(base + i < data_size_); + return *(data_ + base + i); + } + + const Score &data(Count base, Count i = 0) const { + assert(base + i < data_size_); + return *(data_ + base + i); + } + + Count get_count() const { + return count_; + } + + void inc_count() { + count_++; + } + + Count get_distinct() const { + return distinct_; + } + + void inc_distinct() { + distinct_++; + } + + const PhraseText &get_phrase() const { + return phrase_; + } + + void inc_n1() { + n1_++; + } + + Count get_n1() { + return n1_; + } + + void inc_n2() { + n2_++; + } + + Count get_n2() { + return n2_; + } + + void inc_n3plus() { + n3plus_++; + } + + Count get_n3plus() { + return n3plus_; + } + +}; + +inline std::ostream &operator<<(std::ostream &os, const PhraseInfo &pt) +{ + return os << pt.get_phrase(); +} + +class PhraseInfoList +{ +protected: + typedef std::map IDMapType_; + typedef boost::ptr_vector ListType_; + //typedef std::vector > ListType_; + //typedef std::vector ListType_; + typedef std::list StatListType_; + + IDMapType_ idmap_; + ListType_ list_; + StatListType_ statistics_; + boost::object_pool phrase_info_pool_; + + Count data_size_; +public: + typedef ListType_::iterator iterator; + typedef ListType_::const_iterator const_iterator; + typedef ListType_::size_type size_type; + + PhraseInfoList() : data_size_(0) {} + + Phrase index_phrase(const String &s_phr); + DataIndex register_data(Count size); + void attach_statistic(PhraseStatistic &s); + void compute_statistics(); + + PhraseInfo &operator[](Phrase phr) { + return list_[phr]; + } + + iterator begin() { + return list_.begin(); + } + + iterator end() { + return list_.end(); + } + + const_iterator begin() const { + return list_.begin(); + } + + const_iterator end() const { + return list_.end(); + } + + size_type size() const { + return list_.size(); + } + +}; + +class PhraseAlignment +{ + friend std::ostream &operator<<(std::ostream &os, const PhraseAlignment &pa); + +private: + class Alignment + { + friend std::ostream &operator<<(std::ostream &os, const Alignment &pa); + + private: + typedef boost::tuple AlignmentTuple_; + typedef std::map AlignmentMapType_; + typedef std::vector AlignmentVectorType_; + + static AlignmentMapType_ alignment_map_; + static AlignmentVectorType_ alignment_vector_; + + Count slen_, tlen_; + boost::dynamic_bitset matrix_; + + Alignment(Count slen, Count tlen, const String &alignment); + + public: + bool is_aligned(Count s, Count t) const { + assert(t < tlen_); + assert(s < slen_); + return matrix_[t * slen_ + s]; + } + + Count get_source_length() const { + return slen_; + } + + Count get_target_length() const { + return tlen_; + } + + bool operator<(const Alignment &pa) const { + if(slen_ < pa.slen_) return true; + if(tlen_ < pa.tlen_) return true; + return (matrix_ < pa.matrix_); + } + + static Count index_alignment(Count slen, Count tlen, const String &alignment); + + static const Alignment *find(Count index) { + return alignment_vector_[index]; + } + }; + friend std::ostream &operator<<(std::ostream &os, const Alignment &pa); + + const Alignment *alignment_; + bool reverse_; + +public: + PhraseAlignment(Count index, bool reverse = false) : + alignment_(Alignment::find(index)), reverse_(reverse) {} + + bool is_aligned(Count s, Count t) const { + if(!reverse_) + return alignment_->is_aligned(s, t); + else + return alignment_->is_aligned(t, s); + } + + Count get_source_length() const { + if(!reverse_) + return alignment_->get_source_length(); + else + return alignment_->get_target_length(); + } + + Count get_target_length() const { + if(!reverse_) + return alignment_->get_target_length(); + else + return alignment_->get_source_length(); + } + + static Count index_alignment(Count slen, Count tlen, const String &alignment) { + return Alignment::index_alignment(slen, tlen, alignment); + } +}; + +typedef std::map PhrasePairCounts; + +class PhrasePairInfo +{ +protected: + static const Count CONTINUATION_BIT; + + static bool init_phase_; + static Count data_nscores_; + static Count data_ncounts_; + + enum { COUNT_COUNT_IDX = 0, COUNT_FREE_IDX }; // COUNT_FREE_IDX must remain last! + enum { SCORE_FREE_IDX = 0 }; // SCORE_FREE_IDX must remain last! + + Phrase src_, tgt_; + PhrasePairData data_; + bool reverse_; + + void realloc_data(Count nalignments); + +public: + typedef std::vector > AlignmentVector; + + static DataIndex register_score_data(Count size); + static DataIndex register_count_data(Count size); + + PhrasePairInfo(Count src, Count tgt, Count alignment, Count count); + + PhrasePairInfo(Count src, Count tgt, PhrasePairData data, bool reverse = false) : src_(src), tgt_(tgt), data_(data), reverse_(reverse) { + init_phase_ = false; + } + + PhrasePairInfo(const PhrasePairCounts::const_iterator &in) : + src_(in->first.first), tgt_(in->first.second), data_(in->second), reverse_(false) {} + + PhrasePairData get_phrase_pair_data() { + return data_; + } + + Phrase get_src() const { + return !reverse_ ? src_ : tgt_; + } + + Phrase get_tgt() const { + return !reverse_ ? tgt_ : src_; + } + + Count get_count() const { + return count_data(COUNT_COUNT_IDX); + } + + Score &score_data(DataIndex base, DataIndex index = 0) { + return score_data(data_, base, index); + } + + const Score &score_data(DataIndex base, DataIndex index = 0) const { + return score_data(data_, base, index); + } + + Count &count_data(DataIndex base, DataIndex index = 0) { + return count_data(data_, base, index); + } + + const Count &count_data(DataIndex base, DataIndex index = 0) const { + return count_data(data_, base, index); + } + + void inc_count() { + count_data(data_, COUNT_COUNT_IDX)++; + } + + AlignmentVector get_alignments() const; + void add_alignment(Count alignment); + +private: + static Score &score_data(PhrasePairData data, DataIndex base, DataIndex index = 0) { + return *reinterpret_cast(data + (base + index) * sizeof(Score)); + } + + static Count &count_data(PhrasePairData data, DataIndex base, DataIndex index = 0) { + return *reinterpret_cast(data + data_nscores_ * sizeof(Score) + (base + index) * sizeof(Count)); + } + + static const Count COUNTS_PER_ALIGNMENT = 2; + + static Count *alignment_data(PhrasePairData data, Count index) { + return reinterpret_cast(data + data_nscores_ * sizeof(Score) + data_ncounts_ * sizeof(Count) + COUNTS_PER_ALIGNMENT * index * sizeof(Count)); + } + + Count *alignment_data(Count index) { + return alignment_data(data_, index); + } + + const Count *alignment_data(Count index) const { + return alignment_data(data_, index); + } +}; + +class PhraseTable +{ +public: + typedef PhrasePairInfo value_type; + +protected: + typedef std::iterator_traits::value_type InputEntry_; + typedef value_type (*EntryTransformer_)(InputEntry_); + + static value_type pass_entry(InputEntry_ in) { + return PhrasePairInfo(in.first.first, in.first.second, in.second, false); + } + + static value_type swap_src_tgt(InputEntry_ in) { + return PhrasePairInfo(in.first.first, in.first.second, in.second, true); + } + +public: + typedef boost::transform_iterator iterator; + typedef boost::transform_iterator const_iterator; + + virtual ~PhraseTable() {} + + virtual PhraseInfo &get_src_phrase(Phrase src) = 0; + virtual Count n_src_phrases() const = 0; + virtual PhraseInfo &get_tgt_phrase(Phrase tgt) = 0; + virtual Count n_tgt_phrases() const = 0; + virtual PhrasePairCounts &get_joint_counts() = 0; + virtual void attach_src_statistic(PhraseStatistic &s) = 0; + virtual void attach_tgt_statistic(PhraseStatistic &s) = 0; + virtual void compute_phrase_statistics() = 0; + virtual DataIndex register_src_data(Count n) = 0; + virtual DataIndex register_tgt_data(Count n) = 0; + virtual PhraseTable &reverse() = 0; + + virtual iterator begin() = 0; + virtual iterator end() = 0; + virtual iterator find(PhrasePair p) = 0; + virtual iterator find(const PhrasePairCounts::iterator &it) = 0; + + virtual const_iterator begin() const = 0; + virtual const_iterator end() const = 0; + virtual const_iterator find(PhrasePair p) const = 0; + virtual const_iterator find(const PhrasePairCounts::const_iterator &it) const = 0; + + virtual PhrasePairCounts::iterator raw_begin() = 0; + virtual PhrasePairCounts::iterator raw_end() = 0; + virtual PhrasePairCounts::iterator raw_find(PhrasePair p) = 0; + + virtual PhrasePairCounts::const_iterator raw_begin() const = 0; + virtual PhrasePairCounts::const_iterator raw_end() const = 0; + virtual PhrasePairCounts::const_iterator raw_find(PhrasePair p) const = 0; + + /* + static iterator swap_iterator(const iterator &it) { + if(it.functor() == swap_src_tgt) + return boost::make_transform_iterator(it.base(), pass_entry); + else if(it.functor() == pass_entry) + return boost::make_transform_iterator(it.base(), swap_src_tgt); + else + abort(); + } + + static const_iterator swap_iterator(const const_iterator &it) { + if(it.functor() == swap_src_tgt) + return boost::make_transform_iterator(it.base(), pass_entry); + else if(it.functor() == pass_entry) + return boost::make_transform_iterator(it.base(), swap_src_tgt); + else + abort(); + } + */ +}; + +class ReversePhraseTable : public PhraseTable +{ +protected: + PhraseTable &phrase_table_; + +public: + ReversePhraseTable(PhraseTable &phrase_table) : + phrase_table_(phrase_table) {} + + virtual PhraseInfo &get_src_phrase(Phrase src) { + return phrase_table_.get_tgt_phrase(src); + } + + virtual Count n_src_phrases() const { + return phrase_table_.n_tgt_phrases(); + } + + virtual PhraseInfo &get_tgt_phrase(Phrase tgt) { + return phrase_table_.get_src_phrase(tgt); + } + + virtual Count n_tgt_phrases() const { + return phrase_table_.n_src_phrases(); + } + + virtual PhrasePairCounts &get_joint_counts() { + return phrase_table_.get_joint_counts(); + } + + virtual void attach_src_statistic(PhraseStatistic &s) { + return phrase_table_.attach_tgt_statistic(s); + } + + virtual void attach_tgt_statistic(PhraseStatistic &s) { + return phrase_table_.attach_src_statistic(s); + } + + virtual void compute_phrase_statistics() { + phrase_table_.compute_phrase_statistics(); + } + + virtual DataIndex register_src_data(Count n) { + return phrase_table_.register_tgt_data(n); + } + + virtual DataIndex register_tgt_data(Count n) { + return phrase_table_.register_src_data(n); + } + + virtual PhraseTable &reverse() { + return phrase_table_; + } + + virtual iterator begin() { + return boost::make_transform_iterator(phrase_table_.raw_begin(), swap_src_tgt); + } + + virtual iterator end() { + return boost::make_transform_iterator(raw_end(), swap_src_tgt); + } + + virtual iterator find(PhrasePair p) { + return boost::make_transform_iterator(raw_find(p), swap_src_tgt); + } + + virtual iterator find(const PhrasePairCounts::iterator &it) { + return boost::make_transform_iterator(it, swap_src_tgt); + } + + virtual const_iterator begin() const { + return boost::make_transform_iterator(phrase_table_.raw_begin(), swap_src_tgt); + } + + virtual const_iterator end() const { + return boost::make_transform_iterator(raw_end(), swap_src_tgt); + } + + virtual const_iterator find(PhrasePair p) const { + return boost::make_transform_iterator(raw_find(p), swap_src_tgt); + } + + virtual const_iterator find(const PhrasePairCounts::const_iterator &it) const { + return boost::make_transform_iterator(it, swap_src_tgt); + } + + virtual PhrasePairCounts::iterator raw_begin() { + return phrase_table_.raw_begin(); + } + + virtual PhrasePairCounts::iterator raw_end() { + return phrase_table_.raw_end(); + } + + virtual PhrasePairCounts::iterator raw_find(PhrasePair p) { + return phrase_table_.raw_find(std::make_pair(p.second, p.first)); + } + + virtual PhrasePairCounts::const_iterator raw_begin() const { + return phrase_table_.raw_begin(); + } + + virtual PhrasePairCounts::const_iterator raw_end() const { + return phrase_table_.raw_end(); + } + + virtual PhrasePairCounts::const_iterator raw_find(PhrasePair p) const { + return phrase_table_.raw_find(std::make_pair(p.second, p.first)); + } +}; + +class MemoryPhraseTable : public PhraseTable +{ +protected: + PhraseInfoList src_info_; + PhraseInfoList tgt_info_; + PhrasePairCounts joint_counts_; + + ReversePhraseTable reverse_; + +public: + MemoryPhraseTable() : reverse_(*this) {} + + void load_data(std::istream &instream); + + virtual PhraseInfo &get_src_phrase(Phrase src) { + assert(src < src_info_.size()); + return src_info_[src]; + } + + virtual Count n_src_phrases() const { + return src_info_.size(); + } + + virtual PhraseInfo &get_tgt_phrase(Phrase tgt) { + assert(tgt < tgt_info_.size()); + return tgt_info_[tgt]; + } + + virtual Count n_tgt_phrases() const { + return tgt_info_.size(); + } + + virtual PhrasePairCounts &get_joint_counts() { + return joint_counts_; + } + + virtual void attach_src_statistic(PhraseStatistic &s); + virtual void attach_tgt_statistic(PhraseStatistic &s); + virtual void compute_phrase_statistics(); + + virtual DataIndex register_src_data(Count n) { + return src_info_.register_data(n); + } + + virtual DataIndex register_tgt_data(Count n) { + return tgt_info_.register_data(n); + } + + virtual PhraseTable &reverse() { + return reverse_; + } + + virtual iterator begin() { + return boost::make_transform_iterator(raw_begin(), pass_entry); + } + + virtual iterator end() { + return boost::make_transform_iterator(raw_end(), pass_entry); + } + + virtual iterator find(PhrasePair p) { + return boost::make_transform_iterator(raw_find(p), pass_entry); + } + + virtual iterator find(const PhrasePairCounts::iterator &it) { + return boost::make_transform_iterator(it, pass_entry); + } + + virtual const_iterator begin() const { + return boost::make_transform_iterator(raw_begin(), pass_entry); + } + + virtual const_iterator end() const { + return boost::make_transform_iterator(raw_end(), pass_entry); + } + + virtual const_iterator find(PhrasePair p) const { + return boost::make_transform_iterator(raw_find(p), pass_entry); + } + + virtual const_iterator find(const PhrasePairCounts::const_iterator &it) const { + return boost::make_transform_iterator(it, pass_entry); + } + + virtual PhrasePairCounts::iterator raw_begin() { + return joint_counts_.begin(); + } + + virtual PhrasePairCounts::iterator raw_end() { + return joint_counts_.end(); + } + + virtual PhrasePairCounts::iterator raw_find(PhrasePair p) { + return joint_counts_.find(p); + } + + virtual PhrasePairCounts::const_iterator raw_begin() const { + return joint_counts_.begin(); + } + + virtual PhrasePairCounts::const_iterator raw_end() const { + return joint_counts_.end(); + } + + virtual PhrasePairCounts::const_iterator raw_find(PhrasePair p) const { + return joint_counts_.find(p); + } +}; + +#endif diff --git a/mosesdecoder/contrib/memscore/scorer-impl.h b/mosesdecoder/contrib/memscore/scorer-impl.h new file mode 100644 index 0000000000000000000000000000000000000000..bc544f12e86704efcf9c8e07e0c2103dae0f0b38 --- /dev/null +++ b/mosesdecoder/contrib/memscore/scorer-impl.h @@ -0,0 +1,124 @@ +// memscore - in-memory phrase scoring for Statistical Machine Translation +// Christian Hardmeier, FBK-irst, Trento, 2010 +// $Id$ + +#ifndef SCORER_IMPL_H +#define SCORER_IMPL_H + +#include "phrasetable.h" +#include "scorer.h" + +class MLPhraseScorer : public PhraseScorer +{ +private: + explicit MLPhraseScorer(PhraseTable &pd, bool reverse) : + PhraseScorer(pd, reverse) {} + + virtual void do_score_phrases(); + virtual Score do_get_score(const PhraseTable::const_iterator &it); + +public: + static PhraseScorer *create_scorer(const char *argv[], int &argp, bool reverse, const PhraseScorerFactory &ptf); +}; + +class WittenBellPhraseScorer : public PhraseScorer +{ +private: + explicit WittenBellPhraseScorer(PhraseTable &pd, bool reverse) : + PhraseScorer(pd, reverse) {} + + virtual Score do_get_score(const PhraseTable::const_iterator &it); + +public: + static PhraseScorer *create_scorer(const char *argv[], int &argp, bool reverse, const PhraseScorerFactory &ptf); +}; + +class AbsoluteDiscountPhraseScorer : public PhraseScorer +{ +private: + Score discount_; + + explicit AbsoluteDiscountPhraseScorer(PhraseTable &pd, bool reverse) : + PhraseScorer(pd, reverse) {} + + virtual void do_score_phrases(); + virtual Score do_get_score(const PhraseTable::const_iterator &it); + +public: + Score get_discount(); + static PhraseScorer *create_scorer(const char *argv[], int &argp, bool reverse, const PhraseScorerFactory &ptf); +}; + +class KNDiscount1PhraseScorer : public PhraseScorer +{ +private: + Count total_distinct_; + Score discount_; + Count total_count_; + + explicit KNDiscount1PhraseScorer(PhraseTable &pd, bool reverse) : + PhraseScorer(pd, reverse) {} + + virtual void do_score_phrases(); + virtual Score do_get_score(const PhraseTable::const_iterator &it); + +public: + static PhraseScorer *create_scorer(const char *argv[], int &argp, bool reverse, const PhraseScorerFactory &ptf); +}; + +class KNDiscount3PhraseScorer : public PhraseScorer +{ +private: + Score discount1_; + Score discount2_; + Score discount3plus_; + + Count total_distinct_n1_; + Count total_distinct_n2_; + Count total_distinct_n3plus_; + + explicit KNDiscount3PhraseScorer(PhraseTable &pd, bool reverse) : + PhraseScorer(pd, reverse) {} + + virtual void do_score_phrases(); + virtual Score do_get_score(const PhraseTable::const_iterator &it); + +public: + static PhraseScorer *create_scorer(const char *argv[], int &argp, bool reverse, const PhraseScorerFactory &ptf); +}; + +class LexicalWeightPhraseScorer : public PhraseScorer +{ +private: + typedef std::map,Score> WeightMapType_; + + WeightMapType_ weight_map_; + bool overall_max_score_; + const Count null_word_; + + LexicalWeightPhraseScorer(PhraseTable &pd, bool reverse, const String &weights, bool overall_max = true); + + Score get_weight(const String &s_src, const String &s_tgt) const; + Score get_weight(Count src, Count tgt) const; + + virtual void do_score_phrases(); + virtual Score do_get_score(const PhraseTable::const_iterator &it); + +public: + static PhraseScorer *create_scorer(const char *argv[], int &argp, bool reverse, const PhraseScorerFactory &ptf); +}; + +class ConstantPhraseScorer : public PhraseScorer +{ +private: + Score constant_; + + ConstantPhraseScorer(PhraseTable &pt, bool reverse, Score constant) : PhraseScorer(pt, reverse), constant_(constant) {} + + virtual Score do_get_score(const PhraseTable::const_iterator &it); + +public: + static PhraseScorer *create_scorer(const char *argv[], int &argp, bool reverse, const PhraseScorerFactory &ptf); +}; + +#endif diff --git a/mosesdecoder/contrib/promix/README.md b/mosesdecoder/contrib/promix/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1b3d76f2732cbbdcd38aeca2874423cbb09cdcce --- /dev/null +++ b/mosesdecoder/contrib/promix/README.md @@ -0,0 +1,66 @@ +promix - for training translation model interpolation weights using PRO + +Author: Barry Haddow + +ABOUT +----- + +The code here provides the "inner loop" for a batch tuning algorithm (like MERT) which +optimises phrase table interpolation weights at the same time as the standard linear +model weights. Interpolation of the phrase tables uses the "naive" method of tmcombine. + +Currently the interpolation only works for two phrase tables, but will soon +be extended to work for more than two. + + +REQUIREMENTS +------------ +The scripts require the Moses Python interface (in contrib/python). It should be built +first, following the instructions in that directory. + +The scripts also require scipy and numpy. They have been tested with the following versions: + +* Python 2.7 +* Scipy 0.11.0 +* Numpy 1.6.2 + +Run the test.py script to check that everything is functioning correctly. + + +USAGE +----- +Since the code in this directory provides the inner loop for a batch tuning algorithm, +it is run from the increasingly inaccurately named mert-moses.pl. If you want to run +the optimiser directly, run `main.py -h` for usage. + +A sample command for mert-moses.pl is as follows: + + MOSES/scripts/training/mert-moses.pl \ + input-file ref-file \ + decoder \ + ini-file \ + --promix-training MOSES/contrib/promix/main.py \ + --maximum-iterations 15 \ + --promix-table phrase-table-1 \ + --promix-table phrase-table-2 \ + --filtercmd "MOSES/scripts/training/filter-model-given-input.pl --Binarizer MOSES/bin/processPhraseTable" \ + --nbest 100 --working-dir ./tmp --decoder-flags "-threads 4 -v 0 " \ + --rootdir MOSES/scripts -mertdir MOSES/bin \ + --return-best-dev + +Note that promix training requires a filter and binarise script, and that the phrase table +referenced in the ini file is not used. The argument `--return-best-dev` is not essential, +but recommended. + + +REFERENCES +---------- + +The code here was created for: + +Haddow, Barry (2013) Applying Pairwise Ranked Optimisation to +Improve the Interpolation of Translation Models. In: Proceedings of NAACL 2013 + +See also: + +Sennrich, Rico (2012). Perplexity Minimization for Translation Model Domain Adaptation in Statistical Machine Translation. In: Proceedings of EACL 2012. diff --git a/mosesdecoder/contrib/promix/bleu.py b/mosesdecoder/contrib/promix/bleu.py new file mode 100644 index 0000000000000000000000000000000000000000..de68a280010ed5ecf353c860ee8eb0379e7dc0e4 --- /dev/null +++ b/mosesdecoder/contrib/promix/bleu.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python + +from math import exp,log + +class BleuScoreException(Exception): + pass + +class BleuScorer: + def __init__(self): + """References should be a list. If each element is a string, assume + they are filenames, if a list, assume tokenised strings""" + self.smooth = 1.0 + self.order = 4 + + def score(self,scores): + if len(scores) != self.order*2+1: + raise BleuScoreException("Wrong number of scores. Expected %d, but found %d" % + (self.order*2+1, len(scores))) + logbleu = 0.0 + for j in range(self.order): + logbleu += log(scores[2*j] + self.smooth) - log(scores[2*j+1] + self.smooth) + logbleu /= self.order + brevity = 1.0 - float(scores[-1]) / scores[1] + if brevity < 0: + logbleu += brevity + return exp(logbleu) + diff --git a/mosesdecoder/contrib/promix/coll.py b/mosesdecoder/contrib/promix/coll.py new file mode 100644 index 0000000000000000000000000000000000000000..850b488ffd930dc1e9eafcc1b1e3d97c1cf3b192 --- /dev/null +++ b/mosesdecoder/contrib/promix/coll.py @@ -0,0 +1,260 @@ +# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. +# Passes Python2.7's test suite and incorporates all the latest updates. + +# From http://code.activestate.com/recipes/576693/ + +try: + from thread import get_ident as _get_ident +except ImportError: + from dummy_thread import get_ident as _get_ident + +try: + from _abcoll import KeysView, ValuesView, ItemsView +except ImportError: + pass + + +class OrderedDict(dict): + 'Dictionary that remembers insertion order' + # An inherited dict maps keys to values. + # The inherited dict provides __getitem__, __len__, __contains__, and get. + # The remaining methods are order-aware. + # Big-O running times for all methods are the same as for regular dictionaries. + + # The internal self.__map dictionary maps keys to links in a doubly linked list. + # The circular doubly linked list starts and ends with a sentinel element. + # The sentinel element never gets deleted (this simplifies the algorithm). + # Each link is stored as a list of length three: [PREV, NEXT, KEY]. + + def __init__(self, *args, **kwds): + '''Initialize an ordered dictionary. Signature is the same as for + regular dictionaries, but keyword arguments are not recommended + because their insertion order is arbitrary. + + ''' + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + try: + self.__root + except AttributeError: + self.__root = root = [] # sentinel node + root[:] = [root, root, None] + self.__map = {} + self.__update(*args, **kwds) + + def __setitem__(self, key, value, dict_setitem=dict.__setitem__): + 'od.__setitem__(i, y) <==> od[i]=y' + # Setting a new item creates a new link which goes at the end of the linked + # list, and the inherited dictionary is updated with the new key/value pair. + if key not in self: + root = self.__root + last = root[0] + last[1] = root[0] = self.__map[key] = [last, root, key] + dict_setitem(self, key, value) + + def __delitem__(self, key, dict_delitem=dict.__delitem__): + 'od.__delitem__(y) <==> del od[y]' + # Deleting an existing item uses self.__map to find the link which is + # then removed by updating the links in the predecessor and successor nodes. + dict_delitem(self, key) + link_prev, link_next, key = self.__map.pop(key) + link_prev[1] = link_next + link_next[0] = link_prev + + def __iter__(self): + 'od.__iter__() <==> iter(od)' + root = self.__root + curr = root[1] + while curr is not root: + yield curr[2] + curr = curr[1] + + def __reversed__(self): + 'od.__reversed__() <==> reversed(od)' + root = self.__root + curr = root[0] + while curr is not root: + yield curr[2] + curr = curr[0] + + def clear(self): + 'od.clear() -> None. Remove all items from od.' + try: + for node in self.__map.itervalues(): + del node[:] + root = self.__root + root[:] = [root, root, None] + self.__map.clear() + except AttributeError: + pass + dict.clear(self) + + def popitem(self, last=True): + '''od.popitem() -> (k, v), return and remove a (key, value) pair. + Pairs are returned in LIFO order if last is true or FIFO order if false. + + ''' + if not self: + raise KeyError('dictionary is empty') + root = self.__root + if last: + link = root[0] + link_prev = link[0] + link_prev[1] = root + root[0] = link_prev + else: + link = root[1] + link_next = link[1] + root[1] = link_next + link_next[0] = root + key = link[2] + del self.__map[key] + value = dict.pop(self, key) + return key, value + + # -- the following methods do not depend on the internal structure -- + + def keys(self): + 'od.keys() -> list of keys in od' + return list(self) + + def values(self): + 'od.values() -> list of values in od' + return [self[key] for key in self] + + def items(self): + 'od.items() -> list of (key, value) pairs in od' + return [(key, self[key]) for key in self] + + def iterkeys(self): + 'od.iterkeys() -> an iterator over the keys in od' + return iter(self) + + def itervalues(self): + 'od.itervalues -> an iterator over the values in od' + for k in self: + yield self[k] + + def iteritems(self): + 'od.iteritems -> an iterator over the (key, value) items in od' + for k in self: + yield (k, self[k]) + + def update(*args, **kwds): + '''od.update(E, **F) -> None. Update od from dict/iterable E and F. + + If E is a dict instance, does: for k in E: od[k] = E[k] + If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] + Or if E is an iterable of items, does: for k, v in E: od[k] = v + In either case, this is followed by: for k, v in F.items(): od[k] = v + + ''' + if len(args) > 2: + raise TypeError('update() takes at most 2 positional ' + 'arguments (%d given)' % (len(args),)) + elif not args: + raise TypeError('update() takes at least 1 argument (0 given)') + self = args[0] + # Make progressively weaker assumptions about "other" + other = () + if len(args) == 2: + other = args[1] + if isinstance(other, dict): + for key in other: + self[key] = other[key] + elif hasattr(other, 'keys'): + for key in other.keys(): + self[key] = other[key] + else: + for key, value in other: + self[key] = value + for key, value in kwds.items(): + self[key] = value + + __update = update # let subclasses override update without breaking __init__ + + __marker = object() + + def pop(self, key, default=__marker): + '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. + If key is not found, d is returned if given, otherwise KeyError is raised. + + ''' + if key in self: + result = self[key] + del self[key] + return result + if default is self.__marker: + raise KeyError(key) + return default + + def setdefault(self, key, default=None): + 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' + if key in self: + return self[key] + self[key] = default + return default + + def __repr__(self, _repr_running={}): + 'od.__repr__() <==> repr(od)' + call_key = id(self), _get_ident() + if call_key in _repr_running: + return '...' + _repr_running[call_key] = 1 + try: + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, self.items()) + finally: + del _repr_running[call_key] + + def __reduce__(self): + 'Return state information for pickling' + items = [[k, self[k]] for k in self] + inst_dict = vars(self).copy() + for k in vars(OrderedDict()): + inst_dict.pop(k, None) + if inst_dict: + return (self.__class__, (items,), inst_dict) + return self.__class__, (items,) + + def copy(self): + 'od.copy() -> a shallow copy of od' + return self.__class__(self) + + @classmethod + def fromkeys(cls, iterable, value=None): + '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S + and values equal to v (which defaults to None). + + ''' + d = cls() + for key in iterable: + d[key] = value + return d + + def __eq__(self, other): + '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive + while comparison to a regular mapping is order-insensitive. + + ''' + if isinstance(other, OrderedDict): + return len(self)==len(other) and self.items() == other.items() + return dict.__eq__(self, other) + + def __ne__(self, other): + return not self == other + + # -- the following methods are only used in Python 2.7 -- + + def viewkeys(self): + "od.viewkeys() -> a set-like object providing a view on od's keys" + return KeysView(self) + + def viewvalues(self): + "od.viewvalues() -> an object providing a view on od's values" + return ValuesView(self) + + def viewitems(self): + "od.viewitems() -> a set-like object providing a view on od's items" + return ItemsView(self) diff --git a/mosesdecoder/contrib/promix/main.py b/mosesdecoder/contrib/promix/main.py new file mode 100644 index 0000000000000000000000000000000000000000..fe6305335e0afd14039bb43e2bcb31e58db5d5e1 --- /dev/null +++ b/mosesdecoder/contrib/promix/main.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python + +# +# Implementation of PRO training and extensions to train phrase weights +# + +import gzip +import logging +from numpy import array +import optparse +import os.path +import sys + +from nbest import * +from sampler import * +from train import * + + +logging.basicConfig(format = "%(asctime)-15s %(message)s") +log = logging.getLogger('main') +log.setLevel(logging.DEBUG) + +class Config: + def __init__(self): + self.parser = optparse.OptionParser(usage="%prog [options] ") + self.parser.add_option("-t", "--trainer", action="store",\ + dest="trainer", metavar="TYPE", type="choice", choices=("pro","mix"),\ + default="pro",\ + help="type of trainer to run (pro,mix)") + self.parser.add_option("-n", "--nbest", action="append", \ + dest="nbest", metavar="NBEST-FILE",\ + help="nbest output file(s) from decoder") + self.parser.add_option("-S", "--scfile", action="append",\ + dest="score", metavar="SCORE-FILE",\ + help="score file(s) from extractor (in same order as nbests)") + self.parser.add_option("-p", "--phrase-table" , action="append",\ + dest="ttable", metavar="TTABLE",\ + help="ttable to be used in mixture model training") + self.parser.add_option("-i", "--input-file", action="store",\ + dest="input_file", metavar="INPUT-FILE", + help="source text file") + self.parser.add_option("-m", "--moses-bin-dir", action="store",\ + dest="moses_bin_dir", metavar="DIR", + help="directory containing Moses binaries", + default=os.path.expanduser("~/moses/bin")) + self.nbest_files = [] + self.score_files = [] + self.ttables = [] + + def parse(self,args=sys.argv[1:]): + (options,args) = self.parser.parse_args(args) + self.nbest_files = options.nbest + self.score_files = options.score + self.ttables = options.ttable + self.input_file = options.input_file + self.trainer = options.trainer + self.moses_bin_dir = options.moses_bin_dir + if not self.nbest_files: + self.nbest_files = ["data/esen.nc.nbest.segment"] + if not self.score_files: + self.score_files = ["data/esen.nc.scores"] + if len(self.nbest_files) != len(self.score_files): + self.parser.error("Must have equal numbers of score files and nbest files") + if self.trainer == "mix": + if not self.input_file or not self.ttables: + self.parser.error("Need to specify input file and ttables for mix training") + #if len(self.ttables) != 2: + # self.parser.error("Can only train mix model with 2 ttables at the moment") + +def main(): + config = Config() + config.parse() + + samples = [] + sampler = HopkinsMaySampler() + nbests = 0 + for nbest_file,score_data_file in zip(config.nbest_files,config.score_files): + log.debug("nbest: " + nbest_file + "; score:" + score_data_file) + segments = False + if config.trainer == "mix": segments = True + for nbest in get_scored_nbests(nbest_file, score_data_file, config.input_file, segments=segments): + samples += sampler.sample(nbest) + nbests += 1 + log.debug("Samples loaded") + trainer = None + if config.trainer == "mix": + # Add the phrase table scores + scorer = MosesPhraseScorer(config.ttables) + log.debug("Scoring samples...") + for sample in samples: + scorer.add_scores(sample.hyp1) + scorer.add_scores(sample.hyp2) + log.debug("...samples scored") + trainer = MixtureModelTrainer(samples) + elif config.trainer == "pro": + trainer = ProTrainer(samples) + else: assert(0) + log.debug("Starting training...") + weights,mix_weights = trainer.train(debug=False) + log.debug("...training complete") + for i,w in enumerate(weights): + print "F%d %10.8f" % (i,w) + for i,f in enumerate(mix_weights): + for j,w in enumerate(f): + print "M%d_%d %10.8f" % (i,j,w) + + + + + +if __name__ == "__main__": + main() diff --git a/mosesdecoder/contrib/promix/nbest.py b/mosesdecoder/contrib/promix/nbest.py new file mode 100644 index 0000000000000000000000000000000000000000..d5a6fc90d76ad6beb321989cd1e00e5a538e0ed6 --- /dev/null +++ b/mosesdecoder/contrib/promix/nbest.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python + +import gzip +import os +import re +import numpy as np +import sys + +from bleu import BleuScorer +from coll import OrderedDict +# Edit to set moses python path +sys.path.append(os.path.dirname(__file__) + "/../python") +import moses.dictree as binpt + +class DataFormatException(Exception): + pass + +class Hypothesis: + def __init__(self,text,fv,segments=False): + self.alignment = [] #only stored for segmented hypos + self.tokens = [] #only stored for segmented hypos + if not segments: + self.text = text + # Triples of (source-start, source-end, target-end) where segments end positions + # are 1 beyond the last token + else: + # recover segmentation + self.tokens = [] + align_re = re.compile("\|(\d+)-(\d+)\|") + for token in text.split(): + match = align_re.match(token) + if match: + self.alignment.append\ + ((int(match.group(1)), 1+int(match.group(2)), len(self.tokens))) + else: + self.tokens.append(token) + self.text = " ".join(self.tokens) + if not self.alignment: + raise DataFormatException("Expected segmentation information not found in nbest") + + + self.fv = np.array(fv) + self.score = 0 + + def __str__(self): + return "{text=%s fv=%s score=%5.4f}" % (self.text, str(self.fv), self.score) + +class NBestList: + def __init__(self,id): + self.id = id + self.hyps = [] + +# Maps feature ids (short feature names) to their values +_feature_index = {} +def set_feature_start(name,index): + indexes = _feature_index.get(name, [index,0]) + indexes[0] = index + _feature_index[name] = indexes + +def set_feature_end(name,index): + indexes = _feature_index.get(name, [0,index]) + indexes[1] = index + _feature_index[name] = indexes + +def get_feature_index(name): + return _feature_index.get(name, [0,0]) + +def get_nbests(nbest_file, segments=False): + """Iterate through nbest lists""" + if nbest_file.endswith("gz"): + fh = gzip.GzipFile(nbest_file) + else: + fh = open(nbest_file) + lineno = 0 + nbest = None + for line in fh: + fields = line.split(" ||| ") + if len(fields) != 4: + raise DataFormatException("nbest(%d): %s" % (lineno,line)) + (id, text, scores, total) = fields + if nbest and nbest.id != id: + yield nbest + nbest = None + if not nbest: + nbest = NBestList(id) + fv = [] + score_name = None + for score in scores.split(): + if score.endswith(":"): + score = score[:-1] + if score_name: + set_feature_end(score_name,len(fv)) + score_name = score + set_feature_start(score_name,len(fv)) + else: + fv.append(float(score)) + if score_name: set_feature_end(score_name,len(fv)) + hyp = Hypothesis(text[:-1],fv,segments) + nbest.hyps.append(hyp) + if nbest: + yield nbest + +def get_scores(score_data_file): + """Iterate through the score data, returning a set of scores for each sentence""" + scorer = BleuScorer() + fh = open(score_data_file) + lineno = 0 + score_vectors = None + for line in fh: + if line.startswith("SCORES_TXT_BEGIN"): + score_vectors = [] + elif line.startswith("SCORES_TXT_END"): + scores = [scorer.score(score_vector) for score_vector in score_vectors] + yield scores + else: + score_vectors.append([float(i) for i in line[:-1].split()]) + + +def get_scored_nbests(nbest_file, score_data_file, input_file, segments=False): + score_gen = get_scores(score_data_file) + input_gen = None + if input_file: input_gen = open(input_file) + try: + for nbest in get_nbests(nbest_file, segments=segments): + scores = score_gen.next() + if len(scores) != len(nbest.hyps): + raise DataFormatException("Length of nbest %s does not match score list (%d != %d)" % + (nbest.id,len(nbest.hyps), len(scores))) + input_line = None + if input_gen: + input_line = input_gen.next()[:-1] + for hyp,score in zip(nbest.hyps, scores): + hyp.score = score + hyp.input_line = input_line + yield nbest + + except StopIteration: + raise DataFormatException("Score file shorter than nbest list file") + +class PhraseCache: + """An LRU cache for ttable lookups""" + def __init__(self, max_size): + self.max_size = max_size + self.pairs_to_scores = OrderedDict() + + def get(self, source, target): + key = (source,target) + scores = self.pairs_to_scores.get(key,None) + if scores: + # cache hit - update access time + del self.pairs_to_scores[key] + self.pairs_to_scores[key] = scores + return scores + + def add(self,source,target,scores): + key = (source,target) + self.pairs_to_scores[key] = scores + while len(self.pairs_to_scores) > self.max_size: + self.pairs_to_scores.popitem(last=False) + +# +# Should I store full lists of options, or just phrase pairs? +# Should probably store phrase-pairs, but may want to add +# high scoring pairs (say, 20?) when I load the translations +# of a given phrase +# + +class CachedPhraseTable: + def __init__(self,ttable_file,nscores=5,cache_size=20000): + wa = False + if binpt.PhraseDictionaryTree.canLoad(ttable_file,True): + # assume word alignment is included + wa = True + self.ttable = binpt.PhraseDictionaryTree(ttable_file,nscores = nscores,wa = wa, tableLimit=0) + self.cache = PhraseCache(cache_size) + self.nscores = nscores + + def get_scores(self,phrase): + source = " ".join(phrase[0]) + target_tuple = tuple(phrase[1]) + target = " ".join(target_tuple) + scores = self.cache.get(source,target) + if not scores: + # cache miss + scores = [0] * (self.nscores-1) # ignore penalty + entries = self.ttable.query(source, converter=None) + # find correct target + for entry in entries: + if entry.rhs == target_tuple: + scores = entry.scores[:-1] + break + #print "QUERY",source,"|||",target,"|||",scores + self.cache.add(source,target,scores) + #else: + # print "CACHE",source,"|||",target,"|||",scores + return scores + + +class MosesPhraseScorer: + def __init__(self,ttable_files, cache_size=20000): + self.ttables = [] + for ttable_file in ttable_files: + self.ttables.append(CachedPhraseTable(ttable_file, cache_size=cache_size)) + + def add_scores(self, hyp): + """Add the phrase scores to a hypothesis""" + # Collect up the phrase pairs + phrases = [] + source_tokens = hyp.input_line.split() + tgt_st = 0 + if not hyp.alignment: + raise DataFormatException("Alignments missing from: " + str(hyp)) + for src_st,src_end,tgt_end in hyp.alignment: + phrases.append((source_tokens[src_st:src_end], hyp.tokens[tgt_st:tgt_end])) + tgt_st = tgt_end + # Look up the scores + phrase_scores = [] + for ttable in self.ttables: + phrase_scores.append([]) + for phrase in phrases: + phrase_scores[-1].append(ttable.get_scores(phrase)) +# phrase_scores = np.array(phrase_scores) +# eps = np.exp(-100) +# phrase_scores[phrase_scores 0.5) + self.assertTrue(1 / (1+exp(-np.dot(actual,np.array([-1,-2])))) < 0.5) + + def test_xy(self): + """Test pre-calculation of the y_i*x_ij vectors""" + x = np.array([[1,3], [2,8], [1,3]]) + y = np.array([1,1,-1]) + lro = LogisticRegressionOptimiser(x,y) + expected = np.array([[1,3], [2,8], [-1,-3]]) + for i in 0,1,2: + for j in 0,1: + self.assertEqual(lro.xy[i][j], expected[i][j]) +# +class TestMixtureModelTrainer(unittest.TestCase): + + def setUp(self): + # 3 phrase table features, but last one is ignored for interpolation + nbest._feature_index = {"tm" : [0,3], "lm" : [3,4]} + log05 = np.log(0.5) + log03 = np.log(0.3) + log02 = np.log(0.2) + log01 = np.log(0.1) + hyp0 = nbest.Hypothesis("a |0-0| b c |1-2|", [log05, log05, log02, log03], True) + hyp0.input_line = "A B C" + hyp0.score = 3 + # Two ttables, columns correspond to features, rows to phrase pairs + hyp0.phrase_scores = np.array([\ + [[0.2, 0.3],\ + [0.4, 0.3]],\ + [[0, 0.2],\ + [0.4,0.2]]]) + + hyp1 = nbest.Hypothesis("x |0-2|", [log02, log03, log03, log01], True) + hyp1.input_line = "X Y Z" + hyp1.score = 2 + hyp1.phrase_scores = np.array([\ + [[0.1, 0.1]],\ + [[0.8,0.1]]]) + + hyp2 = nbest.Hypothesis("z |0-1| w |2-2| p |3-3|", [log02, log02, log05, log05], True) + hyp2.score = 1 + hyp2.input_line = "M N O" + # phrase_table x phrase_pair x feature + hyp2.phrase_scores = np.array([\ + [[0.1, 0.2],\ + [0.3,0.5],\ + [0.4,0.6]],\ + [[0.1,0.5],\ + [0.6,0.1],\ + [0.2,0.2]]]) + self.samples = [sampler.Sample(hyp0,hyp1), sampler.Sample(hyp1,hyp2)] + self.trainer = MixtureModelTrainer(self.samples) + + def get_phrase_scores(self, hypothesis, iw): + nptest.assert_almost_equal(np.sum(iw, axis=0), np.array([1.0,1.0])) + phrase_probs = hypothesis.phrase_scores + interpolated_probs = np.sum(np.expand_dims(iw,1)*phrase_probs, axis = 0) + + total_probs = np.prod(interpolated_probs, axis = 0) + return util.safelog(total_probs) + + def model_score(self, hypothesis, weights): + # interpolation weights + # ttable x feature + iw = np.array([[weights[-2], weights[-1]], + [1-weights[-2],1-weights[-1]]]) + #print "iw:",iw + phrase_scores = self.get_phrase_scores(hypothesis,iw) + weighted_phrase_scores = weights[:2] * phrase_scores + score = np.sum(weighted_phrase_scores) + + other_score = np.sum(weights[2:4]*hypothesis.fv[2:4]) + return score + other_score + + + def test_objective(self): + # 2 phrase weights, 2 other feature weights, + # 2 interpolation weights (1 per model x 2 phrase features) + weights = np.array([0.2,0.1,0.4,0.5,0.3,0.6]) + actual = self.trainer.objective(weights) + # Expected objective is the sum of the logs of sigmoids of the score differences + # Weighted by 1 if hyp1 > hyp2, -1 otherwise + expected = 0 + for sample in self.samples: + hyp1_model_score = self.model_score(sample.hyp1, weights) + hyp2_model_score = self.model_score(sample.hyp2, weights) + y = 1 + if sample.hyp2.score > sample.hyp1.score: y = -1 + expected -= log(sigmoid(y * (hyp1_model_score - hyp2_model_score))) + # regularisation + expected += 0.5 * self.trainer.alpha * np.dot(weights[:-2], weights[:-2]) + self.assertAlmostEquals(actual,expected) + + def test_gradient_other(self): + # Gradients are just differences in feature vectors + # fv(hypo0)-fv(hyp1), fv(hyp1)-fv(hyp2) + delta_s = np.vstack((self.samples[0].hyp1.fv-self.samples[0].hyp2.fv,\ + self.samples[1].hyp1.fv-self.samples[1].hyp2.fv)) + # feature functions across rows, samples down columns + # choose other features + other_delta_s = delta_s[:,2:] + actual = self.trainer.gradient_other() + nptest.assert_almost_equal(actual,other_delta_s) + + def test_gradient_phrase(self): + iw = np.array([[0.3, 0.4],[0.7,0.6]]) + sample_deltaf_list = [] + for sample in self.samples: + f_A = self.get_phrase_scores(sample.hyp1, iw) + f_B = self.get_phrase_scores(sample.hyp2, iw) + sample_deltaf_list.append(f_A - f_B) + expected = np.vstack(sample_deltaf_list) # samples down, features along + actual = self.trainer.gradient_phrase(iw) + nptest.assert_almost_equal(actual,expected) + + def test_gradient_interp(self): + # The interpolation weights - ttable x feature + iw = np.array([[0.3, 0.4],[0.7,0.6]]) + phrasew = np.array([1,2]) # The phrase weights + num_ttables = iw.shape[0] + num_phrase_features = iw.shape[1] + bysample_list = [] + # Stack up gradients for each sample + for sample in self.samples: + # Get the gradient of the interpolation weights for each + # hypothesis (A and B) in the sample + byhyp = [] + for hyp in [sample.hyp1,sample.hyp2]: + # the weights are flattened. rows of iw joined together, last row omitted + grad_k = np.array([0.0] * ((num_ttables - 1) * num_phrase_features)) + # Iterate through the phrase features + for j,probs in enumerate(np.transpose(hyp.phrase_scores)): + # j is phrase feature index + # probs is phrase-pair, ttable + grad_jk = np.array([0.0] * (len(iw)-1)) + for l,phi in enumerate(probs): + # For each phrase-pair the gradient term for the lambda + # is the probability for this ttable - probability for last ttable + # divided by overall phrase probability + num = phi[:-1] - phi[-1] + denom = np.sum(iw[:,j]*phi) # use interpolation weights for this feature + grad_jk = grad_jk + (num/denom) + self.assertEquals(len(grad_jk), num_ttables-1) + #print "num",num,"denom",denom,"grad_jk",grad_jk + # add gradient in correct place + #print "\n",j,grad_k,phrasew[j]*grad_jk + grad_k[j*(num_ttables-1):(j+1)*(num_ttables-1)] =\ + grad_k[j*(num_ttables-1):(j+1)*(num_ttables-1)] + phrasew[j]*grad_jk + #print "\ngrad_k",grad_k + byhyp.append(grad_k) + bysample_list.append(byhyp[0]-byhyp[1]) + #print "diff: ", bysample_list[-1] + expected = np.vstack(bysample_list) + actual = self.trainer.gradient_interp(iw,phrasew) + nptest.assert_almost_equal(actual,expected, decimal=5) + + def test_gradient(self): + # 2 phrase weights, 2 other feature weights, + # 2 interpolation weights (2 models and 2 tables) + weights = np.array([0.2,0.1,0.4,0.5,0.6,0.3]) + expected = np.array([0.0] * len(weights)) + # Get the gradients + iw = np.array([[weights[-2], weights[-1]], + [1-weights[-2],1-weights[-1]]]) + phrase_g = self.trainer.gradient_phrase(iw) + other_g = self.trainer.gradient_other() + interp_g = self.trainer.gradient_interp(iw,weights[:2]) + for k,sample in enumerate(self.samples): + hyp1_model_score = self.model_score(sample.hyp1, weights) + hyp2_model_score = self.model_score(sample.hyp2, weights) + y = 1 + if sample.hyp2.score > sample.hyp1.score: y = -1 + delta_score = hyp1_model_score - hyp2_model_score + sig_delta_score = sigmoid(-y * delta_score) + # phrase derivative term + expected[:2] -= (phrase_g[k]*sig_delta_score*y) + # other derivative term + expected[2:4] -= (other_g[k]*sig_delta_score*y) + # inter derivative term + expected[-2:] -= (interp_g[k]*sig_delta_score*y) + expected += self.trainer.alpha*np.append(weights[:-2], np.array([0.0,0.0])) + actual = self.trainer.gradient(weights) + nptest.assert_almost_equal(actual,expected) + + def test_split_weights(self): + w = np.array([1,2,3,4,0.2,0.3]) + sw = self.trainer.get_split_weights(w) + self.assertEquals(len(sw),3) + nptest.assert_almost_equal(sw['phrase'], np.array([1,2])) + nptest.assert_almost_equal(sw['other'], np.array([3,4])) + nptest.assert_almost_equal(sw['interp'], \ + np.array([[0.2,0.3], [0.8,0.7]])) + + + def test_train(self): + """Simple test that it runs without errors""" + print "x=",self.trainer.train() + +if __name__ == "__main__": + unittest.main() + +suite = unittest.TestSuite([ + unittest.TestLoader().loadTestsFromTestCase(TestParabaloidOptimiser), + unittest.TestLoader().loadTestsFromTestCase(TestLogisticRegressionOptimiser), + unittest.TestLoader().loadTestsFromTestCase(TestMixtureModelTrainer)]) + + diff --git a/mosesdecoder/contrib/promix/util.py b/mosesdecoder/contrib/promix/util.py new file mode 100644 index 0000000000000000000000000000000000000000..4d604bd84defd1305a4a13877a94703bfc98229b --- /dev/null +++ b/mosesdecoder/contrib/promix/util.py @@ -0,0 +1,12 @@ +#!/usr/bin/python + +import numpy as np + + +floor = np.exp(-100) +def safelog(x): + """Wraps np.log to give it a floor""" + #return np.log(x) + return np.log(np.clip(x,floor,np.inf)) + + diff --git a/mosesdecoder/jam-files/boost-build/build/ac.jam b/mosesdecoder/jam-files/boost-build/build/ac.jam new file mode 100644 index 0000000000000000000000000000000000000000..6768f358c22fc807b81aff06f9aac2fd79f57b02 --- /dev/null +++ b/mosesdecoder/jam-files/boost-build/build/ac.jam @@ -0,0 +1,198 @@ +# Copyright (c) 2010 Vladimir Prus. +# +# Use, modification and distribution is subject to the Boost Software +# License Version 1.0. (See accompanying file LICENSE_1_0.txt or +# http://www.boost.org/LICENSE_1_0.txt) + +import property-set ; +import path ; +import modules ; +import "class" ; +import errors ; +import configure ; + +rule find-include-path ( variable : properties : header + : provided-path ? ) +{ + # FIXME: document which properties affect this function by + # default. + local target-os = [ $(properties).get ] ; + properties = [ property-set.create $(toolset) ] ; + if $($(variable)-$(properties)) + { + return $($(variable)-$(properties)) ; + } + else + { + provided-path ?= [ modules.peek : $(variable) ] ; + includes = $(provided-path) ; + includes += [ $(properties).get ] ; + if [ $(properties).get ] != windows + { + # FIXME: use sysroot + includes += /usr/include ; + } + + local result ; + while ! $(result) && $(includes) + { + local f = [ path.root $(header) $(includes[1]) ] ; + ECHO "Checking " $(f) ; + if [ path.exists $(f) ] + { + result = $(includes[1]) ; + } + else if $(provided-path) + { + errors.user-error "Could not find header" $(header) + : "in the user-specified directory" $(provided-path) ; + } + includes = $(includes[2-]) ; + } + $(variable)-$(properties) = $(result) ; + return $(result) ; + } +} + +rule find-library ( variable : properties : names + : provided-path ? ) +{ + local target-os = [ $(properties).get ] ; + properties = [ property-set.create $(toolset) ] ; + if $($(variable)-$(properties)) + { + return $($(variable)-$(properties)) ; + } + else + { + provided-path ?= [ modules.peek : $(variable) ] ; + paths = $(provided-path) ; + paths += [ $(properties).get ] ; + if [ $(properties).get ] != windows + { + paths += /usr/lib /usr/lib32 /usr/lib64 ; + } + + local result ; + while ! $(result) && $(paths) + { + while ! $(result) && $(names) + { + local f ; + if $(target-os) = windows + { + f = $(paths[1])/$(names[1]).lib ; + if [ path.exists $(f) ] + { + result = $(f) ; + } + } + else + { + # FIXME: check for .a as well, depending on + # the 'link' feature. + f = $(paths[1])/lib$(names[1]).so ; + ECHO "CHECKING $(f) " ; + if [ path.exists $(f) ] + { + result = $(f) ; + } + } + if ! $(result) && $(provided-path) + { + errors.user-error "Could not find either of: " $(names) + : "in the user-specified directory" $(provided-path) ; + + } + names = $(names[2-]) ; + } + paths = $(paths[2-]) ; + } + $(variable)-$(properties) = $(result) ; + return $(result) ; + } +} + +class ac-library : basic-target +{ + import errors ; + import indirect ; + import virtual-target ; + import ac ; + import configure ; + + rule __init__ ( name : project : * : * ) + { + basic-target.__init__ $(name) : $(project) : $(sources) + : $(requirements) ; + + reconfigure $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ; + } + + rule set-header ( header ) + { + self.header = $(header) ; + } + + rule set-default-names ( names + ) + { + self.default-names = $(names) ; + } + + rule reconfigure ( * : * ) + { + ECHO "XXX" $(1) ; + if ! $(1) + { + # This is 'using xxx ;'. Nothing to configure, really. + } + else + { + for i in 1 2 3 4 5 6 7 8 9 + { + # FIXME: this naming is inconsistent with XXX_INCLUDE/XXX_LIBRARY + if ! ( $($(i)[1]) in root include-path library-path library-name condition ) + { + errors.user-error "Invalid named parameter" $($(i)[1]) ; + } + local name = $($(i)[1]) ; + local value = $($(i)[2-]) ; + if $($(name)) && $($(name)) != $(value) + { + errors.user-error "Attempt to change value of '$(name)'" ; + } + $(name) = $(value) ; + } + + include-path ?= $(root)/include ; + library-path ?= $(root)/lib ; + } + } + + rule construct ( name : sources * : property-set ) + { + # FIXME: log results. + local libnames = $(library-name) ; + if ! $(libnames) && ! $(include-path) && ! $(library-path) + { + libnames = [ modules.peek : $(name:U)_NAME ] ; + # Backward compatibility only. + libnames ?= [ modules.peek : $(name:U)_BINARY ] ; + } + libnames ?= $(self.default-names) ; + + local includes = [ + ac.find-include-path $(name:U)_INCLUDE : $(property-set) : $(self.header) : $(include-path) ] ; + local library = [ ac.find-library $(name:U)_LIBRARY : $(property-set) : $(libnames) : $(library-path) ] ; + if $(includes) && $(library) + { + library = [ virtual-target.from-file $(library) : . : $(self.project) ] ; + configure.log-library-search-result $(name) : "found" ; + return [ property-set.create $(includes) $(library) ] ; + } + else + { + configure.log-library-search-result $(name) : "no found" ; + } + } +} + diff --git a/mosesdecoder/jam-files/boost-build/build/project.jam b/mosesdecoder/jam-files/boost-build/build/project.jam new file mode 100644 index 0000000000000000000000000000000000000000..0fae4d2004c0c4845613c9c196dcc65a181e3eed --- /dev/null +++ b/mosesdecoder/jam-files/boost-build/build/project.jam @@ -0,0 +1,1121 @@ +# Copyright 2002, 2003 Dave Abrahams +# Copyright 2002, 2005, 2006 Rene Rivera +# Copyright 2002, 2003, 2004, 2005, 2006 Vladimir Prus +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) + +# Implements project representation and loading. Each project is represented by: +# - a module where all the Jamfile content live. +# - an instance of 'project-attributes' class. +# (given a module name, can be obtained using the 'attributes' rule) +# - an instance of 'project-target' class (from targets.jam) +# (given a module name, can be obtained using the 'target' rule) +# +# Typically, projects are created as result of loading a Jamfile, which is done +# by rules 'load' and 'initialize', below. First, module for Jamfile is loaded +# and new project-attributes instance is created. Some rules necessary for +# project are added to the module (see 'project-rules' module) at the bottom of +# this file. Default project attributes are set (inheriting attributes of parent +# project, if it exists). After that the Jamfile is read. It can declare its own +# attributes using the 'project' rule which will be combined with any already +# set attributes. +# +# The 'project' rule can also declare a project id which will be associated with +# the project module. +# +# There can also be 'standalone' projects. They are created by calling +# 'initialize' on an arbitrary module and not specifying their location. After +# the call, the module can call the 'project' rule, declare main targets and +# behave as a regular project except that, since it is not associated with any +# location, it should not declare targets that are not prebuilt. +# +# The list of all loaded Jamfile is stored in the .project-locations variable. +# It is possible to obtain a module name for a location using the 'module-name' +# rule. Standalone projects are not recorded and can only be referenced using +# their project id. + +import "class" : new ; +import errors ; +import modules ; +import path ; +import print ; +import property-set ; +import sequence ; + + +# Loads the Jamfile at the given location. After loading, project global file +# and Jamfiles needed by the requested one will be loaded recursively. If the +# Jamfile at that location is loaded already, does nothing. Returns the project +# module for the Jamfile. +# +rule load ( jamfile-location ) +{ + if --debug-loading in [ modules.peek : ARGV ] + { + ECHO "Loading Jamfile at" '$(jamfile-location)' ; + } + + local module-name = [ module-name $(jamfile-location) ] ; + # If Jamfile is already loaded, don't try again. + if ! $(module-name) in $(.jamfile-modules) + { + load-jamfile $(jamfile-location) : $(module-name) ; + + # We want to make sure that child project are loaded only after parent + # projects. In particular, because parent projects define attributes + # which are inherited by children, and we don't want children to be + # loaded before parent has defined everything. + # + # While "build-project" and "use-project" can potentially refer to child + # projects from parent projects, we don't immediately load child + # projects when seeing those attributes. Instead, we record the minimal + # information to be used only later. + load-used-projects $(module-name) ; + } + return $(module-name) ; +} + + +rule load-used-projects ( module-name ) +{ + local used = [ modules.peek $(module-name) : .used-projects ] ; + local location = [ attribute $(module-name) location ] ; + import project ; + while $(used) + { + local id = $(used[1]) ; + local where = $(used[2]) ; + + project.use $(id) : [ path.root [ path.make $(where) ] $(location) ] ; + used = $(used[3-]) ; + } +} + + +# Note the use of character groups, as opposed to listing 'Jamroot' and +# 'jamroot'. With the latter, we would get duplicate matches on Windows and +# would have to eliminate duplicates. +JAMROOT ?= [ modules.peek : JAMROOT ] ; +JAMROOT ?= project-root.jam [Jj]amroot [Jj]amroot.jam ; + + +# Loads parent of Jamfile at 'location'. Issues an error if nothing is found. +# +rule load-parent ( location ) +{ + local found = [ path.glob-in-parents $(location) : $(JAMROOT) $(JAMFILE) ] ; + + if ! $(found) + { + ECHO error: Could not find parent for project at '$(location)' ; + EXIT error: Did not find Jamfile.jam or Jamroot.jam in any parent + directory. ; + } + + return [ load $(found[1]:D) ] ; +} + + +# Makes the specified 'module' act as if it were a regularly loaded Jamfile at +# 'location'. Reports an error if a Jamfile has already been loaded for that +# location. +# +rule act-as-jamfile ( module : location ) +{ + if [ module-name $(location) ] in $(.jamfile-modules) + { + errors.error "Jamfile was already loaded for '$(location)'" ; + } + # Set up non-default mapping from location to module. + .module.$(location) = $(module) ; + + # Add the location to the list of project locations so that we don't try to + # reload the same Jamfile in the future. + .jamfile-modules += [ module-name $(location) ] ; + + initialize $(module) : $(location) ; +} + + +# Returns the project module corresponding to the given project-id or plain +# directory name. Returns nothing if such a project can not be found. +# +rule find ( name : current-location ) +{ + local project-module ; + + # Try interpreting name as project id. + if [ path.is-rooted $(name) ] + { + project-module = $($(name).jamfile-module) ; + } + + if ! $(project-module) + { + local location = [ path.root [ path.make $(name) ] $(current-location) ] + ; + + # If no project is registered for the given location, try to load it. + # First see if we have a Jamfile. If not, then see if we might have a + # project root willing to act as a Jamfile. In that case, project root + # must be placed in the directory referred by id. + + project-module = [ module-name $(location) ] ; + if ! $(project-module) in $(.jamfile-modules) + { + if [ path.glob $(location) : $(JAMROOT) $(JAMFILE) ] + { + project-module = [ load $(location) ] ; + } + else + { + project-module = ; + } + } + } + + return $(project-module) ; +} + + +# Returns the name of the module corresponding to 'jamfile-location'. If no +# module corresponds to that location yet, associates the default module name +# with that location. +# +rule module-name ( jamfile-location ) +{ + if ! $(.module.$(jamfile-location)) + { + # Root the path, so that locations are always unambiguous. Without this, + # we can't decide if '../../exe/program1' and '.' are the same paths. + jamfile-location = [ path.root $(jamfile-location) [ path.pwd ] ] ; + .module.$(jamfile-location) = Jamfile<$(jamfile-location)> ; + } + return $(.module.$(jamfile-location)) ; +} + + +# Default patterns to search for the Jamfiles to use for build declarations. +# +JAMFILE = [ modules.peek : JAMFILE ] ; +JAMFILE ?= [Bb]uild.jam [Jj]amfile.v2 [Jj]amfile [Jj]amfile.jam ; + + +# Find the Jamfile at the given location. This returns the exact names of all +# the Jamfiles in the given directory. The optional parent-root argument causes +# this to search not the given directory but the ones above it up to the +# directory given in it. +# +rule find-jamfile ( + dir # The directory(s) to look for a Jamfile. + parent-root ? # Optional flag indicating to search for the parent Jamfile. + : no-errors ? + ) +{ + # Glob for all the possible Jamfiles according to the match pattern. + # + local jamfile-glob = ; + if $(parent-root) + { + if ! $(.parent-jamfile.$(dir)) + { + .parent-jamfile.$(dir) = [ path.glob-in-parents $(dir) : $(JAMFILE) + ] ; + } + jamfile-glob = $(.parent-jamfile.$(dir)) ; + } + else + { + if ! $(.jamfile.$(dir)) + { + .jamfile.$(dir) = [ path.glob $(dir) : $(JAMFILE) ] ; + } + jamfile-glob = $(.jamfile.$(dir)) ; + + } + + local jamfile-to-load = $(jamfile-glob) ; + # Multiple Jamfiles found in the same place. Warn about this and ensure we + # use only one of them. As a temporary convenience measure, if there is + # Jamfile.v2 among found files, suppress the warning and use it. + # + if $(jamfile-to-load[2-]) + { + local v2-jamfiles = [ MATCH (.*[Jj]amfile\\.v2)|(.*[Bb]uild\\.jam) : $(jamfile-to-load) ] ; + + if $(v2-jamfiles) && ! $(v2-jamfiles[2]) + { + jamfile-to-load = $(v2-jamfiles) ; + } + else + { + local jamfile = [ path.basename $(jamfile-to-load[1]) ] ; + ECHO "warning: Found multiple Jamfiles at '"$(dir)"'!" + "Loading the first one: '$(jamfile)'." ; + } + + jamfile-to-load = $(jamfile-to-load[1]) ; + } + + # Could not find it, error. + # + if ! $(no-errors) && ! $(jamfile-to-load) + { + errors.error Unable to load Jamfile. + : Could not find a Jamfile in directory '$(dir)'. + : Attempted to find it with pattern '"$(JAMFILE:J= )"'. + : Please consult the documentation at 'http://www.boost.org'. ; + } + + return $(jamfile-to-load) ; +} + + +# Load a Jamfile at the given directory. Returns nothing. Will attempt to load +# the file as indicated by the JAMFILE patterns. Effect of calling this rule +# twice with the same 'dir' is undefined. +# +local rule load-jamfile ( + dir # The directory of the project Jamfile. + : jamfile-module + ) +{ + # See if the Jamfile is where it should be. + # + local jamfile-to-load = [ path.glob $(dir) : $(JAMROOT) ] ; + if ! $(jamfile-to-load) + { + jamfile-to-load = [ find-jamfile $(dir) ] ; + } + + if $(jamfile-to-load[2]) + { + errors.error "Multiple Jamfiles found at '$(dir)'" + : "Filenames are: " $(jamfile-to-load:D=) ; + } + + # Now load the Jamfile in it's own context. + # The call to 'initialize' may load parent Jamfile, which might have + # 'use-project' statement that causes a second attempt to load the + # same project we're loading now. Checking inside .jamfile-modules + # prevents that second attempt from messing up. + if ! $(jamfile-module) in $(.jamfile-modules) + { + + # Initialize the Jamfile module before loading. + # + initialize $(jamfile-module) : [ path.parent $(jamfile-to-load) ] + : $(jamfile-to-load:BS) ; + + if ! $(jamfile-module) in $(.jamfile-modules) + { + .jamfile-modules += $(jamfile-module) ; + + local saved-project = $(.current-project) ; + + mark-as-user $(jamfile-module) ; + modules.load $(jamfile-module) : [ path.native $(jamfile-to-load) ] : . ; + if [ MATCH ($(JAMROOT)) : $(jamfile-to-load:BS) ] + { + jamfile = [ find-jamfile $(dir) : no-errors ] ; + if $(jamfile) + { + load-aux $(jamfile-module) : [ path.native $(jamfile) ] ; + } + } + + # Now do some checks. + if $(.current-project) != $(saved-project) + { + errors.error "The value of the .current-project variable has magically" + : "changed after loading a Jamfile. This means some of the targets" + : "might be defined in the wrong project." + : "after loading" $(jamfile-module) + : "expected value" $(saved-project) + : "actual value" $(.current-project) ; + } + + if $(.global-build-dir) + { + local id = [ attribute $(jamfile-module) id ] ; + local project-root = [ attribute $(jamfile-module) project-root ] ; + local location = [ attribute $(jamfile-module) location ] ; + + if $(location) && $(project-root) = $(dir) + { + # This is Jamroot. + if ! $(id) + { + ECHO "warning: the --build-dir option was specified" ; + ECHO "warning: but Jamroot at '$(dir)'" ; + ECHO "warning: specified no project id" ; + ECHO "warning: the --build-dir option will be ignored" ; + } + } + } + } + } +} + + +rule mark-as-user ( module-name ) +{ + if USER_MODULE in [ RULENAMES ] + { + USER_MODULE $(module-name) ; + } +} + + +rule load-aux ( module-name : file ) +{ + mark-as-user $(module-name) ; + + module $(module-name) + { + include $(2) ; + local rules = [ RULENAMES $(1) ] ; + IMPORT $(1) : $(rules) : $(1) : $(1).$(rules) ; + } +} + + +.global-build-dir = [ MATCH --build-dir=(.*) : [ modules.peek : ARGV ] ] ; +if $(.global-build-dir) +{ + # If the option is specified several times, take the last value. + .global-build-dir = [ path.make $(.global-build-dir[-1]) ] ; +} + + +# Initialize the module for a project. +# +rule initialize ( + module-name # The name of the project module. + : location ? # The location (directory) of the project to initialize. If + # not specified, a standalone project will be initialized. + : basename ? + ) +{ + if --debug-loading in [ modules.peek : ARGV ] + { + ECHO "Initializing project '$(module-name)'" ; + } + + local jamroot ; + + local parent-module ; + if $(module-name) = test-config + { + # No parent. + } + else if $(module-name) = site-config + { + parent-module = test-config ; + } + else if $(module-name) = user-config + { + parent-module = site-config ; + } + else if $(module-name) = project-config + { + parent-module = user-config ; + } + else + { + # We search for parent/project-root only if Jamfile was specified, i.e. + # if the project is not standalone. + if $(location) && ! [ MATCH ($(JAMROOT)) : $(basename) ] + { + parent-module = [ load-parent $(location) ] ; + } + else + { + # It's either jamroot or standalone project. If it's jamroot, + # inherit from user-config. + if $(location) + { + # If project-config module exist, inherit from it. + if $(project-config.attributes) + { + parent-module = project-config ; + } + else + { + parent-module = user-config ; + } + jamroot = true ; + } + } + } + + # TODO: need to consider if standalone projects can do anything but define + # prebuilt targets. If so, we need to give it a more sensible "location", so + # that source paths are correct. + location ?= "" ; + # Create the module for the Jamfile first. + module $(module-name) + { + } + + # load-parent can end up loading this module again. + # Make sure this isn't duplicated. + if ! $($(module-name).attributes) { + + $(module-name).attributes = [ new project-attributes $(location) + $(module-name) ] ; + local attributes = $($(module-name).attributes) ; + + if $(location) + { + $(attributes).set source-location : [ path.make $(location) ] : exact ; + } + else if ! $(module-name) in test-config site-config user-config project-config + { + # This is a standalone project with known location. Set source location + # so that it can declare targets. This is intended so that you can put + # a .jam file in your sources and use it via 'using'. Standard modules + # (in 'tools' subdir) may not assume source dir is set. + local s = [ modules.binding $(module-name) ] ; + if ! $(s) + { + errors.error "Could not determine project location $(module-name)" ; + } + $(attributes).set source-location : $(s:D) : exact ; + } + + $(attributes).set requirements : [ property-set.empty ] : exact ; + $(attributes).set usage-requirements : [ property-set.empty ] : exact ; + + # Import rules common to all project modules from project-rules module, + # defined at the end of this file. + local rules = [ RULENAMES project-rules ] ; + IMPORT project-rules : $(rules) : $(module-name) : $(rules) ; + + if $(parent-module) + { + inherit-attributes $(module-name) : $(parent-module) ; + $(attributes).set parent-module : $(parent-module) : exact ; + } + + if $(jamroot) + { + $(attributes).set project-root : $(location) : exact ; + } + + local parent ; + if $(parent-module) + { + parent = [ target $(parent-module) ] ; + } + + if ! $(.target.$(module-name)) + { + .target.$(module-name) = [ new project-target $(module-name) + : $(module-name) $(parent) + : [ attribute $(module-name) requirements ] ] ; + + if --debug-loading in [ modules.peek : ARGV ] + { + ECHO "Assigned project target" $(.target.$(module-name)) + "to '$(module-name)'" ; + } + } + } + + .current-project = [ target $(module-name) ] ; +} + + +# Make 'project-module' inherit attributes of project root and parent module. +# +rule inherit-attributes ( project-module : parent-module ) +{ + local attributes = $($(project-module).attributes) ; + local pattributes = [ attributes $(parent-module) ] ; + # Parent module might be locationless configuration module. + if [ modules.binding $(parent-module) ] + { + $(attributes).set parent : [ path.parent + [ path.make [ modules.binding $(parent-module) ] ] ] ; + } + local v = [ $(pattributes).get project-root ] ; + $(attributes).set project-root : $(v) : exact ; + $(attributes).set default-build + : [ $(pattributes).get default-build ] ; + $(attributes).set requirements + : [ $(pattributes).get requirements ] : exact ; + $(attributes).set usage-requirements + : [ $(pattributes).get usage-requirements ] : exact ; + + local parent-build-dir = [ $(pattributes).get build-dir ] ; + if $(parent-build-dir) + { + # Have to compute relative path from parent dir to our dir. Convert both + # paths to absolute, since we cannot find relative path from ".." to + # ".". + + local location = [ attribute $(project-module) location ] ; + local parent-location = [ attribute $(parent-module) location ] ; + + local pwd = [ path.pwd ] ; + local parent-dir = [ path.root $(parent-location) $(pwd) ] ; + local our-dir = [ path.root $(location) $(pwd) ] ; + $(attributes).set build-dir : [ path.join $(parent-build-dir) + [ path.relative $(our-dir) $(parent-dir) ] ] : exact ; + } +} + + +# Associate the given id with the given project module. +# +rule register-id ( id : module ) +{ + $(id).jamfile-module = $(module) ; +} + + +# Class keeping all the attributes of a project. +# +# The standard attributes are "id", "location", "project-root", "parent" +# "requirements", "default-build", "source-location" and "projects-to-build". +# +class project-attributes +{ + import property ; + import property-set ; + import errors ; + import path ; + import print ; + import sequence ; + import project ; + + rule __init__ ( location project-module ) + { + self.location = $(location) ; + self.project-module = $(project-module) ; + } + + # Set the named attribute from the specification given by the user. The + # value actually set may be different. + # + rule set ( attribute : specification * + : exact ? # Sets value from 'specification' without any processing. + ) + { + if $(exact) + { + self.$(attribute) = $(specification) ; + } + else if $(attribute) = "requirements" + { + local result = [ property-set.refine-from-user-input + $(self.requirements) : $(specification) + : $(self.project-module) : $(self.location) ] ; + + if $(result[1]) = "@error" + { + errors.error Requirements for project at '$(self.location)' + conflict with parent's. : Explanation: $(result[2-]) ; + } + else + { + self.requirements = $(result) ; + } + } + else if $(attribute) = "usage-requirements" + { + local unconditional ; + for local p in $(specification) + { + local split = [ property.split-conditional $(p) ] ; + split ?= nothing $(p) ; + unconditional += $(split[2]) ; + } + + local non-free = [ property.remove free : $(unconditional) ] ; + if $(non-free) + { + errors.error usage-requirements $(specification) have non-free + properties $(non-free) ; + } + local t = [ property.translate-paths $(specification) + : $(self.location) ] ; + if $(self.usage-requirements) + { + self.usage-requirements = [ property-set.create + [ $(self.usage-requirements).raw ] $(t) ] ; + } + else + { + self.usage-requirements = [ property-set.create $(t) ] ; + } + } + else if $(attribute) = "default-build" + { + self.default-build = [ property.make $(specification) ] ; + } + else if $(attribute) = "source-location" + { + self.source-location = ; + for local src-path in $(specification) + { + self.source-location += [ path.root [ path.make $(src-path) ] + $(self.location) ] ; + } + } + else if $(attribute) = "build-dir" + { + self.build-dir = [ path.root + [ path.make $(specification) ] $(self.location) ] ; + } + else if $(attribute) = "id" + { + id = [ path.root $(specification) / ] ; + project.register-id $(id) : $(self.project-module) ; + self.id = $(id) ; + } + else if ! $(attribute) in "default-build" "location" "parent" + "projects-to-build" "project-root" "source-location" + { + errors.error Invalid project attribute '$(attribute)' specified for + project at '$(self.location)' ; + } + else + { + self.$(attribute) = $(specification) ; + } + } + + # Returns the value of the given attribute. + # + rule get ( attribute ) + { + return $(self.$(attribute)) ; + } + + # Prints the project attributes. + # + rule print ( ) + { + local id = $(self.id) ; id ?= (none) ; + local parent = $(self.parent) ; parent ?= (none) ; + print.section "'"$(id)"'" ; + print.list-start ; + print.list-item "Parent project:" $(parent) ; + print.list-item "Requirements:" [ $(self.requirements).raw ] ; + print.list-item "Default build:" $(self.default-build) ; + print.list-item "Source location:" $(self.source-location) ; + print.list-item "Projects to build:" + [ sequence.insertion-sort $(self.projects-to-build) ] ; + print.list-end ; + } +} + + +# Returns the project which is currently being loaded. +# +rule current ( ) +{ + return $(.current-project) ; +} + + +# Temporarily changes the current project to 'project'. Should be followed by +# 'pop-current'. +# +rule push-current ( project ) +{ + .saved-current-project += $(.current-project) ; + .current-project = $(project) ; +} + + +rule pop-current ( ) +{ + .current-project = $(.saved-current-project[-1]) ; + .saved-current-project = $(.saved-current-project[1--2]) ; +} + + +# Returns the project-attribute instance for the specified Jamfile module. +# +rule attributes ( project ) +{ + return $($(project).attributes) ; +} + + +# Returns the value of the specified attribute in the specified Jamfile module. +# +rule attribute ( project attribute ) +{ + return [ $($(project).attributes).get $(attribute) ] ; +} + + +# Returns the project target corresponding to the 'project-module'. +# +rule target ( project-module ) +{ + if ! $(.target.$(project-module)) + { + .target.$(project-module) = [ new project-target $(project-module) + : $(project-module) + : [ attribute $(project-module) requirements ] ] ; + } + return $(.target.$(project-module)) ; +} + + +# Use/load a project. +# +rule use ( id : location ) +{ + local saved-project = $(.current-project) ; + local project-module = [ project.load $(location) ] ; + local declared-id = [ project.attribute $(project-module) id ] ; + + if ! $(declared-id) || $(declared-id) != $(id) + { + # The project at 'location' either has no id or that id is not equal to + # the 'id' parameter. + if $($(id).jamfile-module) && ( $($(id).jamfile-module) != + $(project-module) ) + { + errors.user-error Attempt to redeclare already existing project id + '$(id)' + location '$(location)' ; + } + $(id).jamfile-module = $(project-module) ; + } + .current-project = $(saved-project) ; +} + + +# Defines a Boost.Build extension project. Such extensions usually contain +# library targets and features that can be used by many people. Even though +# extensions are really projects, they can be initialized as a module would be +# with the "using" (project.project-rules.using) mechanism. +# +rule extension ( id : options * : * ) +{ + # The caller is a standalone module for the extension. + local mod = [ CALLER_MODULE ] ; + + # We need to do the rest within the extension module. + module $(mod) + { + import path ; + + # Find the root project. + local root-project = [ project.current ] ; + root-project = [ $(root-project).project-module ] ; + while + [ project.attribute $(root-project) parent-module ] && + [ project.attribute $(root-project) parent-module ] != user-config + { + root-project = [ project.attribute $(root-project) parent-module ] ; + } + + # Create the project data, and bring in the project rules into the + # module. + project.initialize $(__name__) : [ path.join [ project.attribute + $(root-project) location ] ext $(1:L) ] ; + + # Create the project itself, i.e. the attributes. All extensions are + # created in the "/ext" project space. + project /ext/$(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : + $(9) ; + local attributes = [ project.attributes $(__name__) ] ; + + # Inherit from the root project of whomever is defining us. + project.inherit-attributes $(__name__) : $(root-project) ; + $(attributes).set parent-module : $(root-project) : exact ; + } +} + + +rule glob-internal ( project : wildcards + : excludes * : rule-name ) +{ + local location = [ $(project).get source-location ] ; + + local result ; + local paths = [ path.$(rule-name) $(location) : + [ sequence.transform path.make : $(wildcards) ] : + [ sequence.transform path.make : $(excludes) ] ] ; + if $(wildcards:D) || $(rule-name) != glob + { + # The paths we have found are relative to the current directory, but the + # names specified in the sources list are assumed to be relative to the + # source directory of the corresponding project. So, just make the names + # absolute. + for local p in $(paths) + { + # If the path is below source location, use relative path. + # Otherwise, use full path just to avoid any ambiguities. + local rel = [ path.relative $(p) $(location) : no-error ] ; + if $(rel) = not-a-child + { + result += [ path.root $(p) [ path.pwd ] ] ; + } + else + { + result += $(rel) ; + } + } + } + else + { + # There were no wildcards in the directory path, so the files are all in + # the source directory of the project. Just drop the directory, instead + # of making paths absolute. + result = $(paths:D="") ; + } + + return $(result) ; +} + + +# This module defines rules common to all projects. +# +module project-rules +{ + rule using ( toolset-module : * ) + { + import toolset ; + import modules ; + import project ; + + # Temporarily change the search path so the module referred to by + # 'using' can be placed in the same directory as Jamfile. User will + # expect the module to be found even though the directory is not in + # BOOST_BUILD_PATH. + local x = [ modules.peek : BOOST_BUILD_PATH ] ; + local caller = [ CALLER_MODULE ] ; + local caller-location = [ modules.binding $(caller) ] ; + modules.poke : BOOST_BUILD_PATH : $(caller-location:D) $(x) ; + toolset.using $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ; + modules.poke : BOOST_BUILD_PATH : $(x) ; + + # The above might have clobbered .current-project. Restore the correct + # value. + modules.poke project : .current-project + : [ project.target $(caller) ] ; + } + + import modules ; + + rule import ( * : * : * ) + { + modules.import project ; + + local caller = [ CALLER_MODULE ] ; + local saved = [ modules.peek project : .current-project ] ; + module $(caller) + { + modules.import $(1) : $(2) : $(3) ; + } + modules.poke project : .current-project : $(saved) ; + } + + rule project ( id ? : options * : * ) + { + import errors ; + import path ; + import project ; + + local caller = [ CALLER_MODULE ] ; + local attributes = [ project.attributes $(caller) ] ; + if $(id) + { + $(attributes).set id : $(id) ; + } + + local explicit-build-dir ; + + for n in 2 3 4 5 6 7 8 9 + { + local option = $($(n)) ; + if $(option) + { + $(attributes).set $(option[1]) : $(option[2-]) ; + } + if $(option[1]) = "build-dir" + { + explicit-build-dir = [ path.make $(option[2-]) ] ; + } + } + + # If '--build-dir' is specified, change the build dir for the project. + local global-build-dir = + [ modules.peek project : .global-build-dir ] ; + + if $(global-build-dir) + { + local location = [ $(attributes).get location ] ; + # Project with an empty location is a 'standalone' project such as + # user-config or qt. It has no build dir. If we try to set build dir + # for user-config, we shall then try to inherit it, with either + # weird or wrong consequences. + if $(location) && $(location) = [ $(attributes).get project-root ] + { + # Re-read the project id, since it might have been changed in + # the project's attributes. + id = [ $(attributes).get id ] ; + # This is Jamroot. + if $(id) + { + if $(explicit-build-dir) && + [ path.is-rooted $(explicit-build-dir) ] + { + errors.user-error Absolute directory specified via + 'build-dir' project attribute : Do not know how to + combine that with the --build-dir option. ; + } + # Strip the leading slash from id. + local rid = [ MATCH /(.*) : $(id) ] ; + local p = [ path.join + $(global-build-dir) $(rid) $(explicit-build-dir) ] ; + + $(attributes).set build-dir : $(p) : exact ; + } + } + else + { + # Not Jamroot. + if $(explicit-build-dir) + { + errors.user-error When --build-dir is specified, the + 'build-dir' project : attribute is allowed only for + top-level 'project' invocations ; + } + } + } + } + + # Declare and set a project global constant. Project global constants are + # normal variables but should not be changed. They are applied to every + # child Jamfile. + # + rule constant ( + name # Variable name of the constant. + : value + # Value of the constant. + ) + { + import project ; + local caller = [ CALLER_MODULE ] ; + local p = [ project.target $(caller) ] ; + $(p).add-constant $(name) : $(value) ; + } + + # Declare and set a project global constant, whose value is a path. The path + # is adjusted to be relative to the invocation directory. The given value + # path is taken to be either absolute, or relative to this project root. + # + rule path-constant ( + name # Variable name of the constant. + : value + # Value of the constant. + ) + { + import project ; + local caller = [ CALLER_MODULE ] ; + local p = [ project.target $(caller) ] ; + $(p).add-constant $(name) : $(value) : path ; + } + + rule use-project ( id : where ) + { + import modules ; + # See comment in 'load' for explanation. + local caller = [ CALLER_MODULE ] ; + modules.poke $(caller) : .used-projects : + [ modules.peek $(caller) : .used-projects ] + $(id) $(where) ; + } + + rule build-project ( dir ) + { + import project ; + local caller = [ CALLER_MODULE ] ; + local attributes = [ project.attributes $(caller) ] ; + + local now = [ $(attributes).get projects-to-build ] ; + $(attributes).set projects-to-build : $(now) $(dir) ; + } + + rule explicit ( target-names * ) + { + import project ; + # If 'explicit' is used in a helper rule defined in Jamroot and + # inherited by children, then most of the time we want 'explicit' to + # operate on the Jamfile where the helper rule is invoked. + local t = [ project.current ] ; + for local n in $(target-names) + { + $(t).mark-target-as-explicit $(n) ; + } + } + + rule always ( target-names * ) + { + import project ; + local t = [ project.current ] ; + for local n in $(target-names) + { + $(t).mark-target-as-always $(n) ; + } + } + + rule glob ( wildcards + : excludes * ) + { + import project ; + return [ project.glob-internal [ project.current ] : $(wildcards) : + $(excludes) : glob ] ; + } + + rule glob-tree ( wildcards + : excludes * ) + { + import project ; + + if $(wildcards:D) || $(excludes:D) + { + errors.user-error The patterns to 'glob-tree' may not include + directory ; + } + return [ project.glob-internal [ project.current ] : $(wildcards) : + $(excludes) : glob-tree ] ; + } + + # Calculates conditional requirements for multiple requirements at once. + # This is a shorthand to reduce duplication and to keep an inline + # declarative syntax. For example: + # + # lib x : x.cpp : [ conditional gcc debug : + # DEBUG_EXCEPTION DEBUG_TRACE ] ; + # + rule conditional ( condition + : requirements * ) + { + local condition = $(condition:J=,) ; + if [ MATCH (:) : $(condition) ] + { + return $(condition)$(requirements) ; + } + else + { + return $(condition):$(requirements) ; + } + } + + rule option ( name : value ) + { + local m = [ CALLER_MODULE ] ; + if $(m) != site-config && $(m) != user-config && $(m) != project-config + { + import errors ; + errors.error "The 'option' rule may be used only in site-config or user-config" ; + } + import option ; + option.set $(name) : $(value) ; + } +} diff --git a/mosesdecoder/jam-files/boost-build/build/property-set.jam b/mosesdecoder/jam-files/boost-build/build/property-set.jam new file mode 100644 index 0000000000000000000000000000000000000000..51e2b20de4ba23c26219df69f2a55af9da129dda --- /dev/null +++ b/mosesdecoder/jam-files/boost-build/build/property-set.jam @@ -0,0 +1,489 @@ +# Copyright 2003 Dave Abrahams +# Copyright 2003, 2004, 2005, 2006 Vladimir Prus +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) + +import "class" : new ; +import feature ; +import path ; +import project ; +import property ; +import sequence ; +import set ; +import option ; + +# Class for storing a set of properties. +# +# There is 1<->1 correspondence between identity and value. No two instances +# of the class are equal. To maintain this property, the 'property-set.create' +# rule should be used to create new instances. Instances are immutable. +# +# Each property is classified with regard to its effect on build results. +# Incidental properties have no effect on build results, from Boost.Build's +# point of view. Others are either free, or non-free and we refer to non-free +# ones as 'base'. Each property belongs to exactly one of those categories. +# +# It is possible to get a list of properties belonging to each category as +# well as a list of properties with a specific attribute. +# +# Several operations, like and refine and as-path are provided. They all use +# caching whenever possible. +# +class property-set +{ + import errors ; + import feature ; + import path ; + import property ; + import property-set ; + import set ; + + rule __init__ ( raw-properties * ) + { + self.raw = $(raw-properties) ; + + for local p in $(raw-properties) + { + if ! $(p:G) + { + errors.error "Invalid property: '$(p)'" ; + } + } + } + + # Returns Jam list of stored properties. + # + rule raw ( ) + { + return $(self.raw) ; + } + + rule str ( ) + { + return "[" $(self.raw) "]" ; + } + + # Returns properties that are neither incidental nor free. + # + rule base ( ) + { + if ! $(self.base-initialized) + { + init-base ; + } + return $(self.base) ; + } + + # Returns free properties which are not incidental. + # + rule free ( ) + { + if ! $(self.base-initialized) + { + init-base ; + } + return $(self.free) ; + } + + # Returns dependency properties. + # + rule dependency ( ) + { + if ! $(self.dependency-initialized) + { + init-dependency ; + } + return $(self.dependency) ; + } + + rule non-dependency ( ) + { + if ! $(self.dependency-initialized) + { + init-dependency ; + } + return $(self.non-dependency) ; + } + + rule conditional ( ) + { + if ! $(self.conditional-initialized) + { + init-conditional ; + } + return $(self.conditional) ; + } + + rule non-conditional ( ) + { + if ! $(self.conditional-initialized) + { + init-conditional ; + } + return $(self.non-conditional) ; + } + + # Returns incidental properties. + # + rule incidental ( ) + { + if ! $(self.base-initialized) + { + init-base ; + } + return $(self.incidental) ; + } + + rule refine ( ps ) + { + if ! $(self.refined.$(ps)) + { + local r = [ property.refine $(self.raw) : [ $(ps).raw ] ] ; + if $(r[1]) != "@error" + { + self.refined.$(ps) = [ property-set.create $(r) ] ; + } + else + { + self.refined.$(ps) = $(r) ; + } + } + return $(self.refined.$(ps)) ; + } + + rule expand ( ) + { + if ! $(self.expanded) + { + self.expanded = [ property-set.create [ feature.expand $(self.raw) ] ] ; + } + return $(self.expanded) ; + } + + rule expand-composites ( ) + { + if ! $(self.composites) + { + self.composites = [ property-set.create + [ feature.expand-composites $(self.raw) ] ] ; + } + return $(self.composites) ; + } + + rule evaluate-conditionals ( context ? ) + { + context ?= $(__name__) ; + if ! $(self.evaluated.$(context)) + { + self.evaluated.$(context) = [ property-set.create + [ property.evaluate-conditionals-in-context $(self.raw) : [ $(context).raw ] ] ] ; + } + return $(self.evaluated.$(context)) ; + } + + rule propagated ( ) + { + if ! $(self.propagated-ps) + { + local result ; + for local p in $(self.raw) + { + if propagated in [ feature.attributes $(p:G) ] + { + result += $(p) ; + } + } + self.propagated-ps = [ property-set.create $(result) ] ; + } + return $(self.propagated-ps) ; + } + + rule add-defaults ( ) + { + if ! $(self.defaults) + { + self.defaults = [ property-set.create + [ feature.add-defaults $(self.raw) ] ] ; + } + return $(self.defaults) ; + } + + rule as-path ( ) + { + if ! $(self.as-path) + { + self.as-path = [ property.as-path [ base ] ] ; + } + return $(self.as-path) ; + } + + # Computes the path to be used for a target with the given properties. + # Returns a list of + # - the computed path + # - if the path is relative to the build directory, a value of 'true'. + # + rule target-path ( ) + { + if ! $(self.target-path) + { + # The feature can be used to explicitly change the + # location of generated targets. + local l = [ get ] ; + if $(l) + { + self.target-path = $(l) ; + } + else + { + local p = [ as-path ] ; + p = [ property-set.hash-maybe $(p) ] ; + + # A real ugly hack. Boost regression test system requires + # specific target paths, and it seems that changing it to handle + # other directory layout is really hard. For that reason, we + # teach V2 to do the things regression system requires. The + # value of '' is prepended to the path. + local prefix = [ get ] ; + if $(prefix) + { + self.target-path = [ path.join $(prefix) $(p) ] ; + } + else + { + self.target-path = $(p) ; + } + if ! $(self.target-path) + { + self.target-path = . ; + } + # The path is relative to build dir. + self.target-path += true ; + } + } + return $(self.target-path) ; + } + + rule add ( ps ) + { + if ! $(self.added.$(ps)) + { + self.added.$(ps) = [ property-set.create $(self.raw) [ $(ps).raw ] ] ; + } + return $(self.added.$(ps)) ; + } + + rule add-raw ( properties * ) + { + return [ add [ property-set.create $(properties) ] ] ; + } + + # Returns all values of 'feature'. + # + rule get ( feature ) + { + if ! $(self.map-built) + { + # For each feature, create a member var and assign all values to it. + # Since all regular member vars start with 'self', there will be no + # conflicts between names. + self.map-built = true ; + for local v in $(self.raw) + { + $(v:G) += $(v:G=) ; + } + } + return $($(feature)) ; + } + + # private + + rule init-base ( ) + { + for local p in $(self.raw) + { + local att = [ feature.attributes $(p:G) ] ; + # A feature can be both incidental and free, in which case we add it + # to incidental. + if incidental in $(att) + { + self.incidental += $(p) ; + } + else if free in $(att) + { + self.free += $(p) ; + } + else + { + self.base += $(p) ; + } + } + self.base-initialized = true ; + } + + rule init-dependency ( ) + { + for local p in $(self.raw) + { + local att = [ feature.attributes $(p:G) ] ; + + if dependency in $(att) + { + self.dependency += $(p) ; + } + else + { + self.non-dependency += $(p) ; + } + } + self.dependency-initialized = true ; + } + + rule init-conditional ( ) + { + for local p in $(self.raw) + { + if [ MATCH (:) : $(p:G=) ] + { + self.conditional += $(p) ; + } + else + { + self.non-conditional += $(p) ; + } + } + self.conditional-initialized = true ; + } +} + + +# Creates a new 'property-set' instance for the given raw properties or returns +# an already existing ones. +# +rule create ( raw-properties * ) +{ + raw-properties = [ sequence.unique + [ sequence.insertion-sort $(raw-properties) ] ] ; + + local key = $(raw-properties:J=-:E=) ; + + if ! $(.ps.$(key)) + { + .ps.$(key) = [ new property-set $(raw-properties) ] ; + } + return $(.ps.$(key)) ; +} +NATIVE_RULE property-set : create ; + + +# Creates a new 'property-set' instance after checking that all properties are +# valid and converting incidental properties into gristed form. +# +rule create-with-validation ( raw-properties * ) +{ + property.validate $(raw-properties) ; + return [ create [ property.make $(raw-properties) ] ] ; +} + + +# Creates a property-set from the input given by the user, in the context of +# 'jamfile-module' at 'location'. +# +rule create-from-user-input ( raw-properties * : jamfile-module location ) +{ + local specification = [ property.translate-paths $(raw-properties) + : $(location) ] ; + specification = [ property.translate-indirect $(specification) + : $(jamfile-module) ] ; + local project-id = [ project.attribute $(jamfile-module) id ] ; + project-id ?= [ path.root $(location) [ path.pwd ] ] ; + specification = [ property.translate-dependencies + $(specification) : $(project-id) : $(location) ] ; + specification = + [ property.expand-subfeatures-in-conditions $(specification) ] ; + specification = [ property.make $(specification) ] ; + return [ property-set.create $(specification) ] ; +} + + +# Refines requirements with requirements provided by the user. Specially handles +# "-value" syntax in specification to remove given requirements. +# - parent-requirements -- property-set object with requirements to refine. +# - specification -- string list of requirements provided by the user. +# - project-module -- module to which context indirect features will be +# bound. +# - location -- path to which path features are relative. +# +rule refine-from-user-input ( parent-requirements : specification * : + project-module : location ) +{ + if ! $(specification) + { + return $(parent-requirements) ; + } + else + { + local add-requirements ; + local remove-requirements ; + + for local r in $(specification) + { + local m = [ MATCH "^-(.*)" : $(r) ] ; + if $(m) + { + remove-requirements += $(m) ; + } + else + { + add-requirements += $(r) ; + } + } + + if $(remove-requirements) + { + # Need to create a property set, so that path features and indirect + # features are translated just like they are in project + # requirements. + local ps = [ property-set.create-from-user-input + $(remove-requirements) : $(project-module) $(location) ] ; + + parent-requirements = [ property-set.create + [ set.difference [ $(parent-requirements).raw ] + : [ $(ps).raw ] ] ] ; + specification = $(add-requirements) ; + } + + local requirements = [ property-set.create-from-user-input + $(specification) : $(project-module) $(location) ] ; + + return [ $(parent-requirements).refine $(requirements) ] ; + } +} + + +# Returns a property-set with an empty set of properties. +# +rule empty ( ) +{ + if ! $(.empty) + { + .empty = [ create ] ; + } + return $(.empty) ; +} + +if [ option.get hash : : yes ] = yes +{ + rule hash-maybe ( path ? ) + { + path ?= "" ; + return [ MD5 $(path) ] ; + } +} +else +{ + rule hash-maybe ( path ? ) + { + return $(path) ; + } +} + diff --git a/mosesdecoder/jam-files/boost-build/build/targets.jam b/mosesdecoder/jam-files/boost-build/build/targets.jam new file mode 100644 index 0000000000000000000000000000000000000000..12e4a5edffd623c78ddea526751ad06ad890e419 --- /dev/null +++ b/mosesdecoder/jam-files/boost-build/build/targets.jam @@ -0,0 +1,1659 @@ +# Copyright Vladimir Prus 2002. +# Copyright Rene Rivera 2006. +# +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt) + +# Supports 'abstract' targets, which are targets explicitly defined in a +# Jamfile. +# +# Abstract targets are represented by classes derived from 'abstract-target' +# class. The first abstract target is 'project-target', which is created for +# each Jamfile, and can be obtained by the 'target' rule in the Jamfile's module +# (see project.jam). +# +# Project targets keep a list of 'main-target' instances. A main target is what +# the user explicitly defines in a Jamfile. It is possible to have several +# definitions for a main target, for example to have different lists of sources +# for different platforms. So, main targets keep a list of alternatives. +# +# Each alternative is an instance of 'abstract-target'. When a main target +# subvariant is defined by some rule, that rule will decide what class to use, +# create an instance of that class and add it to the list of alternatives for +# the main target. +# +# Rules supplied by the build system will use only targets derived from +# 'basic-target' class, which will provide some default behaviour. There will be +# different classes derived from it such as 'make-target', created by the 'make' +# rule, and 'typed-target', created by rules such as 'exe' and 'lib'. + +# +# +------------------------+ +# |abstract-target | +# +========================+ +# |name | +# |project | +# | | +# |generate(properties) = 0| +# +-----------+------------+ +# | +# ^ +# / \ +# +-+-+ +# | +# | +# +------------------------+------+------------------------------+ +# | | | +# | | | +# +----------+-----------+ +------+------+ +------+-------+ +# | project-target | | main-target | | basic-target | +# +======================+ 1 * +=============+ alternatives +==============+ +# | generate(properties) |o-----------+ generate |<>------------->| generate | +# | main-target | +-------------+ | construct = 0| +# +----------------------+ +--------------+ +# | +# ^ +# / \ +# +-+-+ +# | +# | +# ...--+----------------+------------------+----------------+---+ +# | | | | +# | | | | +# ... ---+-----+ +------+-------+ +------+------+ +--------+-----+ +# | | typed-target | | make-target | | stage-target | +# . +==============+ +=============+ +==============+ +# . | construct | | construct | | construct | +# +--------------+ +-------------+ +--------------+ + +import assert ; +import "class" : new ; +import errors ; +import feature ; +import indirect ; +import path ; +import property ; +import property-set ; +import sequence ; +import set ; +import toolset ; +import build-request ; + + +# Base class for all abstract targets. +# +class abstract-target +{ + import project ; + import assert ; + import "class" ; + import errors ; + + rule __init__ ( name # Name of the target in Jamfile. + : project-target # The project target to which this one belongs. + ) + { + # Note: it might seem that we don't need either name or project at all. + # However, there are places where we really need it. One example is + # error messages which should name problematic targets. Another is + # setting correct paths for sources and generated files. + + self.name = $(name) ; + self.project = $(project-target) ; + self.location = [ errors.nearest-user-location ] ; + } + + # Returns the name of this target. + rule name ( ) + { + return $(self.name) ; + } + + # Returns the project for this target. + rule project ( ) + { + return $(self.project) ; + } + + # Return the location where the target was declared. + rule location ( ) + { + return $(self.location) ; + } + + # Returns a user-readable name for this target. + rule full-name ( ) + { + local location = [ $(self.project).get location ] ; + return $(location)/$(self.name) ; + } + + # Generates virtual targets for this abstract target using the specified + # properties, unless a different value of some feature is required by the + # target. + # On success, returns: + # - a property-set with the usage requirements to be applied to dependants + # - a list of produced virtual targets, which may be empty. + # If 'property-set' is empty, performs the default build of this target, in + # a way specific to the derived class. + # + rule generate ( property-set ) + { + errors.error "method should be defined in derived classes" ; + } + + rule rename ( new-name ) + { + self.name = $(new-name) ; + } +} + + +if --debug-building in [ modules.peek : ARGV ] +{ + modules.poke : .debug-building : true ; +} + + +rule indent ( ) +{ + return $(.indent:J="") ; +} + + +rule increase-indent ( ) +{ + .indent += " " ; +} + + +rule decrease-indent ( ) +{ + .indent = $(.indent[2-]) ; +} + + +# Project target class (derived from 'abstract-target'). +# +# This class has the following responsibilities: +# - Maintaining a list of main targets in this project and building them. +# +# Main targets are constructed in two stages: +# - When Jamfile is read, a number of calls to 'add-alternative' is made. At +# that time, alternatives can also be renamed to account for inline targets. +# - The first time 'main-target' or 'has-main-target' rule is called, all +# alternatives are enumerated and main targets are created. +# +class project-target : abstract-target +{ + import project ; + import targets ; + import path ; + import print ; + import property-set ; + import set ; + import sequence ; + import "class" : new ; + import errors ; + + rule __init__ ( name : project-module parent-project ? + : requirements * : default-build * ) + { + abstract-target.__init__ $(name) : $(__name__) ; + + self.project-module = $(project-module) ; + self.location = [ project.attribute $(project-module) location ] ; + self.requirements = $(requirements) ; + self.default-build = $(default-build) ; + + if $(parent-project) + { + inherit $(parent-project) ; + } + } + + # This is needed only by the 'make' rule. Need to find the way to make + # 'make' work without this method. + # + rule project-module ( ) + { + return $(self.project-module) ; + } + + rule get ( attribute ) + { + return [ project.attribute $(self.project-module) $(attribute) ] ; + } + + rule build-dir ( ) + { + if ! $(self.build-dir) + { + self.build-dir = [ get build-dir ] ; + if ! $(self.build-dir) + { + self.build-dir = [ path.join [ $(self.project).get location ] + bin ] ; + } + } + return $(self.build-dir) ; + } + + # Generates all possible targets contained in this project. + # + rule generate ( property-set * ) + { + if [ modules.peek : .debug-building ] + { + ECHO [ targets.indent ] "building project" [ name ] " ('$(__name__)') with" [ $(property-set).raw ] ; + targets.increase-indent ; + } + + local usage-requirements = [ property-set.empty ] ; + local targets ; + + for local t in [ targets-to-build ] + { + local g = [ $(t).generate $(property-set) ] ; + usage-requirements = [ $(usage-requirements).add $(g[1]) ] ; + targets += $(g[2-]) ; + } + targets.decrease-indent ; + return $(usage-requirements) [ sequence.unique $(targets) ] ; + } + + # Computes and returns a list of abstract-target instances which must be + # built when this project is built. + # + rule targets-to-build ( ) + { + local result ; + + if ! $(self.built-main-targets) + { + build-main-targets ; + } + + # Collect all main targets here, except for "explicit" ones. + for local t in $(self.main-targets) + { + if ! [ $(t).name ] in $(self.explicit-targets) + { + result += $(t) ; + } + } + + # Collect all projects referenced via "projects-to-build" attribute. + local self-location = [ get location ] ; + for local pn in [ get projects-to-build ] + { + result += [ find $(pn)/ ] ; + } + + return $(result) ; + } + + # Add 'target' to the list of targets in this project that should be build + # only by explicit request + # + rule mark-target-as-explicit ( target-name * ) + { + # Record the name of the target, not instance, since this rule is called + # before main target instances are created. + self.explicit-targets += $(target-name) ; + } + + rule mark-target-as-always ( target-name * ) + { + # Record the name of the target, not instance, since this rule is called + # before main target instances are created. + self.always-targets += $(target-name) ; + } + + # Add new target alternative + # + rule add-alternative ( target-instance ) + { + if $(self.built-main-targets) + { + errors.error add-alternative called when main targets are already + created. : in project [ full-name ] ; + } + self.alternatives += $(target-instance) ; + } + + # Returns a 'main-target' class instance corresponding to 'name'. + # + rule main-target ( name ) + { + if ! $(self.built-main-targets) + { + build-main-targets ; + } + return $(self.main-target.$(name)) ; + } + + # Returns whether a main target with the specified name exists. + # + rule has-main-target ( name ) + { + if ! $(self.built-main-targets) + { + build-main-targets ; + } + + if $(self.main-target.$(name)) + { + return true ; + } + } + + # Worker function for the find rule not implementing any caching and simply + # returning nothing in case the target can not be found. + # + rule find-really ( id ) + { + local result ; + local current-location = [ get location ] ; + + local split = [ MATCH (.*)//(.*) : $(id) ] ; + local project-part = $(split[1]) ; + local target-part = $(split[2]) ; + + local extra-error-message ; + if $(project-part) + { + # There is an explicitly specified project part in id. Looks up the + # project and passes the request to it. + local pm = [ project.find $(project-part) : $(current-location) ] ; + if $(pm) + { + project-target = [ project.target $(pm) ] ; + result = [ $(project-target).find $(target-part) : no-error ] ; + } + else + { + # TODO: This extra error message will not get displayed most + # likely due to some buggy refactoring. Refactor the code so the + # message gets diplayed again. + extra-error-message = error: could not find project + '$(project-part)' ; + } + } + else + { + # Interpret target-name as name of main target. Need to do this + # before checking for file. Consider the following scenario with a + # toolset not modifying its executable's names, e.g. gcc on + # Unix-like platforms: + # + # exe test : test.cpp ; + # install s : test : . ; + # + # After the first build we would have a target named 'test' in the + # Jamfile and a file named 'test' on the disk. We need the target to + # override the file. + result = [ main-target $(id) ] ; + + # Interpret id as an existing file reference. + if ! $(result) + { + result = [ new file-reference [ path.make $(id) ] : + $(self.project) ] ; + if ! [ $(result).exists ] + { + result = ; + } + } + + # Interpret id as project-id. + if ! $(result) + { + local project-module = [ project.find $(id) : + $(current-location) ] ; + if $(project-module) + { + result = [ project.target $(project-module) ] ; + } + } + } + + return $(result) ; + } + + # Find and return the target with the specified id, treated relative to + # self. Id may specify either a target or a file name with the target taking + # priority. May report an error or return nothing if the target is not found + # depending on the 'no-error' parameter. + # + rule find ( id : no-error ? ) + { + local v = $(.id.$(id)) ; + if ! $(v) + { + v = [ find-really $(id) ] ; + if ! $(v) + { + v = none ; + } + .id.$(id) = $(v) ; + } + + if $(v) != none + { + return $(v) ; + } + else + { + if ! $(no-error) + { + local current-location = [ get location ] ; + ECHO "error: Unable to find file or target named" ; + ECHO "error: '$(id)'" ; + ECHO "error: referred from project at" ; + ECHO "error: '$(current-location)'" ; + ECHO $(extra-error-message) ; + EXIT ; + } + } + } + + rule build-main-targets ( ) + { + self.built-main-targets = true ; + for local a in $(self.alternatives) + { + local name = [ $(a).name ] ; + local target = $(self.main-target.$(name)) ; + if ! $(target) + { + local t = [ new main-target $(name) : $(self.project) ] ; + self.main-target.$(name) = $(t) ; + self.main-targets += $(t) ; + target = $(self.main-target.$(name)) ; + } + + if $(name) in $(self.always-targets) + { + $(a).always ; + } + + $(target).add-alternative $(a) ; + } + } + + # Accessor, add a constant. + # + rule add-constant ( + name # Variable name of the constant. + : value + # Value of the constant. + : type ? # Optional type of value. + ) + { + switch $(type) + { + case path : + local r ; + for local v in $(value) + { + local l = $(self.location) ; + if ! $(l) + { + # Project corresponding to config files do not have + # 'location' attribute, but do have source location. + # It might be more reasonable to make every project have + # a location and use some other approach to prevent buildable + # targets in config files, but that's for later. + l = [ get source-location ] ; + } + v = [ path.root [ path.make $(v) ] $(l) ] ; + # Now make the value absolute path. + v = [ path.root $(v) [ path.pwd ] ] ; + # Constants should be in platform-native form. + v = [ path.native $(v) ] ; + r += $(v) ; + } + value = $(r) ; + } + if ! $(name) in $(self.constants) + { + self.constants += $(name) ; + } + self.constant.$(name) = $(value) ; + # Inject the constant in the scope of the Jamroot module. + modules.poke $(self.project-module) : $(name) : $(value) ; + } + + rule inherit ( parent ) + { + for local c in [ modules.peek $(parent) : self.constants ] + { + # No need to pass the type. Path constants were converted to + # absolute paths already by parent. + add-constant $(c) + : [ modules.peek $(parent) : self.constant.$(c) ] ; + } + + # Import rules from parent. + local this-module = [ project-module ] ; + local parent-module = [ $(parent).project-module ] ; + # Do not import rules coming from 'project-rules' as they must be + # imported localized. + local user-rules = [ set.difference + [ RULENAMES $(parent-module) ] : + [ RULENAMES project-rules ] ] ; + IMPORT $(parent-module) : $(user-rules) : $(this-module) : $(user-rules) ; + EXPORT $(this-module) : $(user-rules) ; + } +} + + +# Helper rules to detect cycles in main target references. +# +local rule start-building ( main-target-instance ) +{ + if $(main-target-instance) in $(.targets-being-built) + { + local names ; + for local t in $(.targets-being-built) $(main-target-instance) + { + names += [ $(t).full-name ] ; + } + + errors.error "Recursion in main target references" + : "the following target are being built currently:" + : $(names) ; + } + .targets-being-built += $(main-target-instance) ; +} + + +local rule end-building ( main-target-instance ) +{ + .targets-being-built = $(.targets-being-built[1--2]) ; +} + + +# A named top-level target in Jamfile. +# +class main-target : abstract-target +{ + import assert ; + import errors ; + import feature ; + import print ; + import property-set ; + import sequence ; + import targets : start-building end-building ; + + rule __init__ ( name : project ) + { + abstract-target.__init__ $(name) : $(project) ; + } + + # Add a new alternative for this target + rule add-alternative ( target ) + { + local d = [ $(target).default-build ] ; + if $(self.alternatives) && ( $(self.default-build) != $(d) ) + { + errors.error "default build must be identical in all alternatives" + : "main target is" [ full-name ] + : "with" [ $(d).raw ] + : "differing from previous default build" [ $(self.default-build).raw ] ; + } + else + { + self.default-build = $(d) ; + } + self.alternatives += $(target) ; + } + + # Returns the best viable alternative for this property-set. See the + # documentation for selection rules. + # + local rule select-alternatives ( property-set debug ? ) + { + # When selecting alternatives we have to consider defaults, for example: + # lib l : l.cpp : debug ; + # lib l : l_opt.cpp : release ; + # won't work unless we add default value debug. + property-set = [ $(p).add-defaults ] ; + + # The algorithm: we keep the current best viable alternative. When we've + # got a new best viable alternative, we compare it with the current one. + + local best ; + local best-properties ; + + if $(self.alternatives[2-]) + { + local bad ; + local worklist = $(self.alternatives) ; + while $(worklist) && ! $(bad) + { + local v = $(worklist[1]) ; + local properties = [ $(v).match $(property-set) $(debug) ] ; + + if $(properties) != no-match + { + if ! $(best) + { + best = $(v) ; + best-properties = $(properties) ; + } + else + { + if $(properties) = $(best-properties) + { + bad = true ; + } + else if $(properties) in $(best-properties) + { + # Do nothing, this alternative is worse + } + else if $(best-properties) in $(properties) + { + best = $(v) ; + best-properties = $(properties) ; + } + else + { + bad = true ; + } + } + } + worklist = $(worklist[2-]) ; + } + if ! $(bad) + { + return $(best) ; + } + } + else + { + return $(self.alternatives) ; + } + } + + rule apply-default-build ( property-set ) + { + return [ targets.apply-default-build $(property-set) + : $(self.default-build) ] ; + } + + # Select an alternative for this main target, by finding all alternatives + # which requirements are satisfied by 'properties' and picking the one with + # the longest requirements set. Returns the result of calling 'generate' on + # that alternative. + # + rule generate ( property-set ) + { + start-building $(__name__) ; + + # We want composite properties in build request act as if all the + # properties it expands too are explicitly specified. + property-set = [ $(property-set).expand ] ; + + local all-property-sets = [ apply-default-build $(property-set) ] ; + local usage-requirements = [ property-set.empty ] ; + local result ; + for local p in $(all-property-sets) + { + local r = [ generate-really $(p) ] ; + if $(r) + { + usage-requirements = [ $(usage-requirements).add $(r[1]) ] ; + result += $(r[2-]) ; + } + } + end-building $(__name__) ; + return $(usage-requirements) [ sequence.unique $(result) ] ; + } + + # Generates the main target with the given property set and returns a list + # which first element is property-set object containing usage-requirements + # of generated target and with generated virtual target in other elements. + # It is possible that no targets are generated. + # + local rule generate-really ( property-set ) + { + local best-alternatives = [ select-alternatives $(property-set) ] ; + if ! $(best-alternatives) + { + ECHO "error: No best alternative for" [ full-name ] ; + select-alternatives $(property-set) debug ; + return [ property-set.empty ] ; + } + else + { + # Now return virtual targets for the only alternative. + return [ $(best-alternatives).generate $(property-set) ] ; + } + } + + rule rename ( new-name ) + { + abstract-target.rename $(new-name) ; + for local a in $(self.alternatives) + { + $(a).rename $(new-name) ; + } + } +} + + +# Abstract target refering to a source file. This is an artificial entity +# allowing sources to a target to be represented using a list of abstract target +# instances. +# +class file-reference : abstract-target +{ + import virtual-target ; + import property-set ; + import path ; + + rule __init__ ( file : project ) + { + abstract-target.__init__ $(file) : $(project) ; + } + + rule generate ( properties ) + { + return [ property-set.empty ] [ virtual-target.from-file $(self.name) : + [ location ] : $(self.project) ] ; + } + + # Returns true if the referred file really exists. + rule exists ( ) + { + location ; + return $(self.file-path) ; + } + + # Returns the location of target. Needed by 'testing.jam'. + rule location ( ) + { + if ! $(self.file-location) + { + local source-location = [ $(self.project).get source-location ] ; + for local src-dir in $(source-location) + { + if ! $(self.file-location) + { + local location = [ path.root $(self.name) $(src-dir) ] ; + if [ CHECK_IF_FILE [ path.native $(location) ] ] + { + self.file-location = $(src-dir) ; + self.file-path = $(location) ; + } + } + } + } + return $(self.file-location) ; + } +} + + +# Given a target-reference, made in context of 'project', returns the +# abstract-target instance that is referred to, as well as properties explicitly +# specified for this reference. +# +rule resolve-reference ( target-reference : project ) +{ + # Separate target name from properties override. + local split = [ MATCH "^([^<]*)(/(<.*))?$" : $(target-reference) ] ; + local id = $(split[1]) ; + local sproperties = ; + if $(split[3]) + { + sproperties = [ property.make [ feature.split $(split[3]) ] ] ; + sproperties = [ feature.expand-composites $(sproperties) ] ; + } + + # Find the target. + local target = [ $(project).find $(id) ] ; + + return $(target) [ property-set.create $(sproperties) ] ; +} + + +# Attempts to generate the target given by target reference, which can refer +# both to a main target or to a file. Returns a list consisting of +# - usage requirements +# - generated virtual targets, if any +# +rule generate-from-reference ( + target-reference # Target reference. + : project # Project where the reference is made. + : property-set # Properties of the main target that makes the reference. +) +{ + local r = [ resolve-reference $(target-reference) : $(project) ] ; + local target = $(r[1]) ; + local sproperties = $(r[2]) ; + + # Take properties which should be propagated and refine them with + # source-specific requirements. + local propagated = [ $(property-set).propagated ] ; + local rproperties = [ $(propagated).refine $(sproperties) ] ; + if $(rproperties[1]) = "@error" + { + errors.error + "When building" [ full-name ] " with properties " $(properties) : + "Invalid properties specified for " $(source) ":" + $(rproperties[2-]) ; + } + return [ $(target).generate $(rproperties) ] ; +} + +rule apply-default-build ( property-set : default-build ) +{ + # 1. First, see what properties from default-build are already present + # in property-set. + + local raw = [ $(property-set).raw ] ; + local specified-features = $(raw:G) ; + + local defaults-to-apply ; + for local d in [ $(default-build).raw ] + { + if ! $(d:G) in $(specified-features) + { + defaults-to-apply += $(d) ; + } + } + + # 2. If there are any defaults to be applied, form a new build request. + # Pass it through to 'expand-no-defaults' since default-build might + # contain "release debug" resulting in two property-sets. + local result ; + if $(defaults-to-apply) + { + properties = [ + build-request.expand-no-defaults + + # We have to compress subproperties here to prevent property + # lists like: + # + # msvc 7.1 multi + # + # from being expanded into: + # + # 7.1/multi + # msvc/7.1/multi + # + # due to a cross-product property combination. That may be an + # indication that build-request.expand-no-defaults is the wrong + # rule to use here. + [ feature.compress-subproperties $(raw) ] + $(defaults-to-apply) + ] ; + + if $(properties) + { + for local p in $(properties) + { + result += [ property-set.create + [ feature.expand [ feature.split $(p) ] ] ] ; + } + } + else + { + result = [ property-set.empty ] ; + } + } + else + { + result = $(property-set) ; + } + return $(result) ; +} + + +# Given a build request and requirements, return properties common to dependency +# build request and target requirements. +# +# TODO: Document exactly what 'common properties' are, whether they should +# include default property values, whether they should contain any conditional +# properties or should those be already processed, etc. See whether there are +# any differences between use cases with empty and non-empty build-request as +# well as with requirements containing and those not containing any non-free +# features. +# +rule common-properties ( build-request requirements ) +{ + # For optimization, we add free requirements directly, without using a + # complex algorithm. This gives the complex algorithm a better chance of + # caching results. + local free = [ $(requirements).free ] ; + local non-free = [ property-set.create [ $(requirements).base ] + [ $(requirements).incidental ] ] ; + + local key = .rp.$(build-request)-$(non-free) ; + if ! $($(key)) + { + $(key) = [ common-properties2 $(build-request) $(non-free) ] ; + } + return [ $($(key)).add-raw $(free) ] ; +} + + +# Given a 'context' -- a set of already present properties, and 'requirements', +# decide which extra properties should be applied to 'context'. For conditional +# requirements, this means evaluating the condition. For indirect conditional +# requirements, this means calling a rule. Ordinary requirements are always +# applied. +# +# Handles the situation where evaluating one conditional requirement affects +# conditions of another conditional requirements, such as: +# gcc:release release:RELEASE +# +# If 'what' is 'refined' returns context refined with new requirements. If +# 'what' is 'added' returns just the requirements to be applied. +# +rule evaluate-requirements ( requirements : context : what ) +{ + # Apply non-conditional requirements. It is possible that further + # conditional requirement change a value set by non-conditional + # requirements. For example: + # + # exe a : a.cpp : single foo:multi ; + # + # I am not sure if this should be an error, or not, especially given that + # + # single + # + # might come from project's requirements. + + local unconditional = [ feature.expand [ $(requirements).non-conditional ] ] ; + + local raw = [ $(context).raw ] ; + raw = [ property.refine $(raw) : $(unconditional) ] ; + + # We have collected properties that surely must be present in common + # properties. We now try to figure out what other properties should be added + # in order to satisfy rules (4)-(6) from the docs. + + local conditionals = [ $(requirements).conditional ] ; + # The 'count' variable has one element for each conditional feature and for + # each occurrence of '' feature. It is used as a loop + # counter: for each iteration of the loop before we remove one element and + # the property set should stabilize before we are done. It is assumed that + # #conditionals iterations should be enough for properties to propagate + # along conditions in any direction. + local count = $(conditionals) + [ $(requirements).get ] + and-once-more ; + + local added-requirements ; + + local current = $(raw) ; + + # It is assumed that ordinary conditional requirements can not add + # properties (a.k.a. indirect conditional properties), and + # that rules referred to by properties can not add new + # properties. So the list of indirect conditionals does not + # change. + local indirect = [ $(requirements).get ] ; + indirect = [ MATCH ^@(.*) : $(indirect) ] ; + + local ok ; + while $(count) + { + # Evaluate conditionals in context of current properties. + local e = [ property.evaluate-conditionals-in-context $(conditionals) + : $(current) ] ; + + # Evaluate indirect conditionals. + for local i in $(indirect) + { + e += [ indirect.call $(i) $(current) ] ; + } + + if $(e) = $(added-requirements) + { + # If we got the same result, we have found the final properties. + count = ; + ok = true ; + } + else + { + # Oops, conditional evaluation results have changed. Also 'current' + # contains leftovers from a previous evaluation. Recompute 'current' + # using initial properties and conditional requirements. + added-requirements = $(e) ; + current = [ property.refine $(raw) : [ feature.expand $(e) ] ] ; + } + count = $(count[2-]) ; + } + if ! $(ok) + { + errors.error "Can not evaluate conditional properties " $(conditionals) ; + } + + if $(what) = added + { + return [ property-set.create $(unconditional) $(added-requirements) ] ; + } + else if $(what) = refined + { + return [ property-set.create $(current) ] ; + } + else + { + errors.error "Invalid value of the 'what' parameter." ; + } +} + + +rule common-properties2 ( build-request requirements ) +{ + # This guarantees that default properties are present in the result, unless + # they are overriden by some requirement. FIXME: There is possibility that + # we have added bar, which is composite and expands to bar2, but + # default value of is not bar2, in which case it is not clear what to + # do. + # + build-request = [ $(build-request).add-defaults ] ; + # Features added by 'add-default' can be composite and expand to features + # without default values -- so they are not added yet. It could be clearer/ + # /faster to expand only newly added properties but that is not critical. + build-request = [ $(build-request).expand ] ; + + return [ evaluate-requirements $(requirements) : $(build-request) : + refined ] ; +} + +rule push-target ( target ) +{ + .targets = $(target) $(.targets) ; +} + +rule pop-target ( ) +{ + .targets = $(.targets[2-]) ; +} + +# Return the metatarget that is currently being generated. +rule current ( ) +{ + return $(.targets[1]) ; +} + + +# Implements the most standard way of constructing main target alternative from +# sources. Allows sources to be either file or other main target and handles +# generation of those dependency targets. +# +class basic-target : abstract-target +{ + import build-request ; + import build-system ; + import "class" : new ; + import errors ; + import feature ; + import property ; + import property-set ; + import sequence ; + import set ; + import targets ; + import virtual-target ; + + rule __init__ ( name : project : sources * : requirements * + : default-build * : usage-requirements * ) + { + abstract-target.__init__ $(name) : $(project) ; + + self.sources = $(sources) ; + if ! $(requirements) { + requirements = [ property-set.empty ] ; + } + self.requirements = $(requirements) ; + if ! $(default-build) + { + default-build = [ property-set.empty ] ; + } + self.default-build = $(default-build) ; + if ! $(usage-requirements) + { + usage-requirements = [ property-set.empty ] ; + } + self.usage-requirements = $(usage-requirements) ; + + if $(sources:G) + { + errors.user-error properties found in the 'sources' parameter for + [ full-name ] ; + } + } + + rule always ( ) + { + self.always = 1 ; + } + + # Returns the list of abstract-targets which are used as sources. The extra + # properties specified for sources are not represented. The only user for + # this rule at the moment is the "--dump-tests" feature of the test system. + # + rule sources ( ) + { + if ! $(self.source-targets) + { + for local s in $(self.sources) + { + self.source-targets += + [ targets.resolve-reference $(s) : $(self.project) ] ; + } + } + return $(self.source-targets) ; + } + + rule requirements ( ) + { + return $(self.requirements) ; + } + + rule default-build ( ) + { + return $(self.default-build) ; + } + + # Returns the alternative condition for this alternative, if the condition + # is satisfied by 'property-set'. + # + rule match ( property-set debug ? ) + { + # The condition is composed of all base non-conditional properties. It + # is not clear if we should expand 'self.requirements' or not. For one + # thing, it would be nice to be able to put + # msvc-6.0 + # in requirements. On the other hand, if we have release as a + # condition it does not make sense to require full to be + # in the build request just to select this variant. + local bcondition = [ $(self.requirements).base ] ; + local ccondition = [ $(self.requirements).conditional ] ; + local condition = [ set.difference $(bcondition) : $(ccondition) ] ; + if $(debug) + { + ECHO " next alternative: required properties:" $(condition:E=(empty)) ; + } + + if $(condition) in [ $(property-set).raw ] + { + if $(debug) + { + ECHO " matched" ; + } + return $(condition) ; + } + else + { + if $(debug) + { + ECHO " not matched" ; + } + return no-match ; + } + } + + # Takes a target reference, which might be either target id or a dependency + # property, and generates that target using 'property-set' as build request. + # + # The results are added to the variable called 'result-var'. Usage + # requirements are added to the variable called 'usage-requirements-var'. + # + rule generate-dependencies ( dependencies * : property-set + : result-var usage-requirements-var ) + { + for local dependency in $(dependencies) + { + local grist = $(dependency:G) ; + local id = $(dependency:G=) ; + + local result = [ targets.generate-from-reference $(id) : + $(self.project) : $(property-set) ] ; + + $(result-var) += $(result[2-]:G=$(grist)) ; + $(usage-requirements-var) += [ $(result[1]).raw ] ; + } + } + + # Determines final build properties, generates sources, and calls + # 'construct'. This method should not be overridden. + # + rule generate ( property-set ) + { + if [ modules.peek : .debug-building ] + { + ECHO ; + local fn = [ full-name ] ; + ECHO [ targets.indent ] "Building target '$(fn)'" ; + targets.increase-indent ; + ECHO [ targets.indent ] "Build request: " $(property-set) [ $(property-set).raw ] ; + local cf = [ build-system.command-line-free-features ] ; + ECHO [ targets.indent ] "Command line free features: " [ $(cf).raw ] ; + ECHO [ targets.indent ] "Target requirements: " [ $(self.requirements).raw ] ; + } + targets.push-target $(__name__) ; + + if ! $(self.generated.$(property-set)) + { + # Apply free features from the command line. If user said + # define=FOO + # he most likely wants this define to be set for all compiles. + property-set = [ $(property-set).refine + [ build-system.command-line-free-features ] ] ; + local rproperties = [ targets.common-properties $(property-set) + $(self.requirements) ] ; + + if [ modules.peek : .debug-building ] + { + ECHO ; + ECHO [ targets.indent ] "Common properties: " [ $(rproperties).raw ] ; + } + + if ( $(rproperties[1]) != "@error" ) && ( [ $(rproperties).get + ] != no ) + { + local source-targets ; + local properties = [ $(rproperties).non-dependency ] ; + local usage-requirements ; + + generate-dependencies [ $(rproperties).dependency ] : + $(rproperties) : properties usage-requirements ; + + generate-dependencies $(self.sources) : $(rproperties) : + source-targets usage-requirements ; + + if [ modules.peek : .debug-building ] + { + ECHO ; + ECHO [ targets.indent ] "Usage requirements for" + $(self.name)": " $(usage-requirements) ; + } + + rproperties = [ property-set.create $(properties) + $(usage-requirements) ] ; + usage-requirements = [ property-set.create $(usage-requirements) ] ; + + if [ modules.peek : .debug-building ] + { + ECHO [ targets.indent ] "Build properties: " + [ $(rproperties).raw ] ; + } + + local extra = [ $(rproperties).get ] ; + source-targets += $(extra:G=) ; + # We might get duplicate sources, for example if we link to two + # libraries having the same usage requirement. + # Use stable sort, since for some targets the order is + # important. E.g. RUN_PY target need python source to come + # first. + source-targets = [ sequence.unique $(source-targets) : stable ] ; + + local result = [ construct $(self.name) : $(source-targets) : + $(rproperties) ] ; + + if $(result) + { + local gur = $(result[1]) ; + result = $(result[2-]) ; + + if $(self.always) + { + for local t in $(result) + { + $(t).always ; + } + } + + local s = [ create-subvariant $(result) + : [ virtual-target.recent-targets ] + : $(property-set) : $(source-targets) + : $(rproperties) : $(usage-requirements) ] ; + virtual-target.clear-recent-targets ; + + local ur = [ compute-usage-requirements $(s) ] ; + ur = [ $(ur).add $(gur) ] ; + $(s).set-usage-requirements $(ur) ; + if [ modules.peek : .debug-building ] + { + ECHO [ targets.indent ] "Usage requirements from" + $(self.name)": " [ $(ur).raw ] ; + } + + self.generated.$(property-set) = $(ur) $(result) ; + } + } + else + { + if $(rproperties[1]) = "@error" + { + ECHO [ targets.indent ] "Skipping build of:" [ full-name ] + "cannot compute common properties" ; + } + else if [ $(rproperties).get ] = no + { + # If we just see no, we cannot produce any reasonable + # diagnostics. The code that adds this property is expected + # to explain why a target is not built, for example using + # the configure.log-component-configuration function. + } + else + { + ECHO [ targets.indent ] "Skipping build of: " [ full-name ] + " unknown reason" ; + } + + # We are here either because there has been an error computing + # properties or there is no in properties. In the latter + # case we do not want any diagnostic. In the former case, we + # need diagnostics. FIXME + + # If this target fails to build, add no to properties to + # cause any parent target to fail to build. Except that it + # - does not work now, since we check for no only in + # common properties, but not in properties that came from + # dependencies + # - it is not clear if that is a good idea anyway. The alias + # target, for example, should not fail to build if a + # dependency fails. + self.generated.$(property-set) = [ property-set.create no ] ; + } + } + else + { + if [ modules.peek : .debug-building ] + { + ECHO [ targets.indent ] "Already built" ; + local ur = $(self.generated.$(property-set)) ; + ur = $(ur[0]) ; + targets.increase-indent ; + ECHO [ targets.indent ] "Usage requirements from" + $(self.name)": " [ $(ur).raw ] ; + targets.decrease-indent ; + } + } + + targets.pop-target ; + targets.decrease-indent ; + return $(self.generated.$(property-set)) ; + } + + # Given the set of generated targets, and refined build properties, + # determines and sets appropriate usage requirements on those targets. + # + rule compute-usage-requirements ( subvariant ) + { + local rproperties = [ $(subvariant).build-properties ] ; + xusage-requirements = [ targets.evaluate-requirements + $(self.usage-requirements) : $(rproperties) : added ] ; + + # We generate all dependency properties and add them, as well as their + # usage requirements, to the result. + local extra ; + generate-dependencies [ $(xusage-requirements).dependency ] : + $(rproperties) : extra extra ; + + local result = [ property-set.create + [ $(xusage-requirements).non-dependency ] $(extra) ] ; + + # Propagate usage requirements we got from sources, except for the + # and features. + # + # That feature specifies which pch file to use, and should apply only to + # direct dependents. Consider: + # + # pch pch1 : ... + # lib lib1 : ..... pch1 ; + # pch pch2 : + # lib lib2 : pch2 lib1 ; + # + # Here, lib2 should not get property from pch1. + # + # Essentially, when those two features are in usage requirements, they + # are propagated only to direct dependents. We might need a more general + # mechanism, but for now, only those two features are special. + # + # TODO - Actually there are more possible candidates like for instance + # when listing static library X as a source for another static library. + # Then static library X will be added as a property to the + # second library's usage requirements but those requirements should last + # only up to the first executable or shared library that actually links + # to it. + local raw = [ $(subvariant).sources-usage-requirements ] ; + raw = [ $(raw).raw ] ; + raw = [ property.change $(raw) : ] ; + raw = [ property.change $(raw) : ] ; + return [ $(result).add [ property-set.create $(raw) ] ] ; + } + + # Creates new subvariant instances for 'targets'. + # 'root-targets' - virtual targets to be returned to dependants + # 'all-targets' - virtual targets created while building this main target + # 'build-request' - property-set instance with requested build properties + # + local rule create-subvariant ( root-targets * : all-targets * : + build-request : sources * : rproperties : usage-requirements ) + { + for local e in $(root-targets) + { + $(e).root true ; + } + + # Process all virtual targets that will be created if this main target + # is created. + local s = [ new subvariant $(__name__) : $(build-request) : $(sources) : + $(rproperties) : $(usage-requirements) : $(all-targets) ] ; + for local v in $(all-targets) + { + if ! [ $(v).creating-subvariant ] + { + $(v).creating-subvariant $(s) ; + } + } + return $(s) ; + } + + # Constructs virtual targets for this abstract target and the dependency + # graph. Returns a usage-requirements property-set and a list of virtual + # targets. Should be overriden in derived classes. + # + rule construct ( name : source-targets * : properties * ) + { + errors.error "method should be defined in derived classes" ; + } +} + + +class typed-target : basic-target +{ + import generators ; + + rule __init__ ( name : project : type : sources * : requirements * : + default-build * : usage-requirements * ) + { + basic-target.__init__ $(name) : $(project) : $(sources) : + $(requirements) : $(default-build) : $(usage-requirements) ; + + self.type = $(type) ; + } + + rule type ( ) + { + return $(self.type) ; + } + + rule construct ( name : source-targets * : property-set ) + { + local r = [ generators.construct $(self.project) $(name:S=) : $(self.type) + : [ property-set.create [ $(property-set).raw ] + $(self.type) ] + : $(source-targets) : true ] ; + if ! $(r) + { + ECHO "warn: Unable to construct" [ full-name ] ; + + # Are there any top-level generators for this type/property set. + if ! [ generators.find-viable-generators $(self.type) + : $(property-set) ] + { + ECHO "error: no generators were found for type '$(self.type)'" ; + ECHO "error: and the requested properties" ; + ECHO "error: make sure you've configured the needed tools" ; + ECHO "See http://boost.org/boost-build2/doc/html/bbv2/advanced/configuration.html" ; + ECHO "To debug this problem, try the --debug-generators option." ; + EXIT ; + } + } + return $(r) ; + } +} + + +# Return the list of sources to use, if main target rule is invoked with +# 'sources'. If there are any objects in 'sources', they are treated as main +# target instances, and the name of such targets are adjusted to be +# '__'. Such renaming is disabled if +# a non-empty value is passed as the 'no-renaming' parameter. +# +rule main-target-sources ( sources * : main-target-name : no-renaming ? ) +{ + local result ; + for local t in $(sources) + { + if [ class.is-instance $(t) ] + { + local name = [ $(t).name ] ; + if ! $(no-renaming) + { + name = $(main-target-name)__$(name) ; + $(t).rename $(name) ; + } + # Inline targets are not built by default. + local p = [ $(t).project ] ; + $(p).mark-target-as-explicit $(name) ; + result += $(name) ; + } + else + { + result += $(t) ; + } + } + return $(result) ; +} + + +# Returns the requirements to use when declaring a main target, obtained by +# translating all specified property paths and refining project requirements +# with the ones specified for the target. +# +rule main-target-requirements ( + specification * # Properties explicitly specified for the main target. + : project # Project where the main target is to be declared. +) +{ + specification += [ toolset.requirements ] ; + + local requirements = [ property-set.refine-from-user-input + [ $(project).get requirements ] : $(specification) : + [ $(project).project-module ] : [ $(project).get location ] ] ; + if $(requirements[1]) = "@error" + { + errors.error "Conflicting requirements for target:" $(requirements) ; + } + return $(requirements) ; +} + + +# Returns the usage requirements to use when declaring a main target, which are +# obtained by translating all specified property paths and adding project's +# usage requirements. +# +rule main-target-usage-requirements ( + specification * # Use-properties explicitly specified for a main target. + : project # Project where the main target is to be declared. +) +{ + local project-usage-requirements = [ $(project).get usage-requirements ] ; + + # We do not use 'refine-from-user-input' because: + # - I am not sure if removing parent's usage requirements makes sense + # - refining usage requirements is not needed, since usage requirements are + # always free. + local usage-requirements = [ property-set.create-from-user-input + $(specification) + : [ $(project).project-module ] [ $(project).get location ] ] ; + + return [ $(project-usage-requirements).add $(usage-requirements) ] ; +} + + +# Return the default build value to use when declaring a main target, which is +# obtained by using the specified value if not empty and parent's default build +# attribute otherwise. +# +rule main-target-default-build ( + specification * # Default build explicitly specified for a main target. + : project # Project where the main target is to be declared. +) +{ + local result ; + if $(specification) + { + result = $(specification) ; + } + else + { + result = [ $(project).get default-build ] ; + } + return [ property-set.create-with-validation $(result) ] ; +} + + +# Registers the specified target as a main target alternative and returns it. +# +rule main-target-alternative ( target ) +{ + local ptarget = [ $(target).project ] ; + $(ptarget).add-alternative $(target) ; + return $(target) ; +} + +# Creates a new metargets with the specified properties, using 'klass' as +# the class. The 'name', 'sources', +# 'requirements', 'default-build' and 'usage-requirements' are assumed to be in +# the form specified by the user in Jamfile corresponding to 'project'. +# +rule create-metatarget ( klass : project : name : sources * : requirements * : + default-build * : usage-requirements * ) +{ + return [ + targets.main-target-alternative + [ new $(klass) $(name) : $(project) + : [ targets.main-target-sources $(sources) : $(name) ] + : [ targets.main-target-requirements $(requirements) : $(project) ] + : [ targets.main-target-default-build $(default-build) : $(project) ] + : [ targets.main-target-usage-requirements $(usage-requirements) : $(project) ] + ] ] ; +} + +# Creates a typed-target with the specified properties. The 'name', 'sources', +# 'requirements', 'default-build' and 'usage-requirements' are assumed to be in +# the form specified by the user in Jamfile corresponding to 'project'. +# +rule create-typed-target ( type : project : name : sources * : requirements * : + default-build * : usage-requirements * ) +{ + return [ + targets.main-target-alternative + [ new typed-target $(name) : $(project) : $(type) + : [ targets.main-target-sources $(sources) : $(name) ] + : [ targets.main-target-requirements $(requirements) : $(project) ] + : [ targets.main-target-default-build $(default-build) : $(project) ] + : [ targets.main-target-usage-requirements $(usage-requirements) : $(project) ] + ] ] ; +} diff --git a/mosesdecoder/jam-files/boost-build/build/version.jam b/mosesdecoder/jam-files/boost-build/build/version.jam new file mode 100644 index 0000000000000000000000000000000000000000..512c9ee31af7747ab42b5beccfc334e05ecc6cb8 --- /dev/null +++ b/mosesdecoder/jam-files/boost-build/build/version.jam @@ -0,0 +1,161 @@ +# Copyright 2002, 2003, 2004, 2006 Vladimir Prus +# Copyright 2008 Jurko Gospodnetic +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) + +import errors ; +import numbers ; + +major = "2011" ; +minor = "12" ; + +rule boost-build ( ) +{ + return "$(major).$(minor)-svn" ; +} + +rule print ( ) +{ + if [ verify-engine-version ] + { + ECHO "Boost.Build" [ boost-build ] ; + } +} + +rule verify-engine-version ( ) +{ + local v = [ modules.peek : JAM_VERSION ] ; + + if $(v[1]) != $(major) || $(v[2]) != $(minor) + { + local argv = [ modules.peek : ARGV ] ; + local e = $(argv[1]) ; + local l = [ modules.binding version ] ; + l = $(l:D) ; + l = $(l:D) ; + ECHO "warning: mismatched versions of Boost.Build engine and core" ; + ECHO "warning: Boost.Build engine ($(e)) is $(v:J=.)" ; + ECHO "warning: Boost.Build core (at $(l)) is" [ boost-build ] ; + } + else + { + return true ; + } +} + + + +# Utility rule for testing whether all elements in a sequence are equal to 0. +# +local rule is-all-zeroes ( sequence * ) +{ + local result = "true" ; + for local e in $(sequence) + { + if $(e) != "0" + { + result = "" ; + } + } + return $(result) ; +} + + +# Returns "true" if the first version is less than the second one. +# +rule version-less ( lhs + : rhs + ) +{ + numbers.check $(lhs) ; + numbers.check $(rhs) ; + + local done ; + local result ; + + while ! $(done) && $(lhs) && $(rhs) + { + if [ numbers.less $(lhs[1]) $(rhs[1]) ] + { + done = "true" ; + result = "true" ; + } + else if [ numbers.less $(rhs[1]) $(lhs[1]) ] + { + done = "true" ; + } + else + { + lhs = $(lhs[2-]) ; + rhs = $(rhs[2-]) ; + } + } + if ( ! $(done) && ! $(lhs) && ! [ is-all-zeroes $(rhs) ] ) + { + result = "true" ; + } + + return $(result) ; +} + + +# Returns "true" if the current JAM version version is at least the given +# version. +# +rule check-jam-version ( version + ) +{ + local version-tag = $(version:J=.) ; + if ! $(version-tag) + { + errors.error Invalid version specifier: : $(version:E="(undefined)") ; + } + + if ! $(.jam-version-check.$(version-tag))-is-not-empty + { + local jam-version = [ modules.peek : JAM_VERSION ] ; + if ! $(jam-version) + { + errors.error "Unable to deduce Boost Jam version. Your Boost Jam" + "installation is most likely terribly outdated." ; + } + .jam-version-check.$(version-tag) = "true" ; + if [ version-less [ modules.peek : JAM_VERSION ] : $(version) ] + { + .jam-version-check.$(version-tag) = "" ; + } + } + return $(.jam-version-check.$(version-tag)) ; +} + + +rule __test__ ( ) +{ + import assert ; + + local jam-version = [ modules.peek : JAM_VERSION ] ; + local future-version = $(jam-version) ; + future-version += "1" ; + + assert.true check-jam-version $(jam-version) ; + assert.false check-jam-version $(future-version) ; + + assert.true version-less 0 : 1 ; + assert.false version-less 0 : 0 ; + assert.true version-less 1 : 2 ; + assert.false version-less 1 : 1 ; + assert.false version-less 2 : 1 ; + assert.true version-less 3 1 20 : 3 4 10 ; + assert.false version-less 3 1 10 : 3 1 10 ; + assert.false version-less 3 4 10 : 3 1 20 ; + assert.true version-less 3 1 20 5 1 : 3 4 10 ; + assert.false version-less 3 1 10 5 1 : 3 1 10 ; + assert.false version-less 3 4 10 5 1 : 3 1 20 ; + assert.true version-less 3 1 20 : 3 4 10 5 1 ; + assert.true version-less 3 1 10 : 3 1 10 5 1 ; + assert.false version-less 3 4 10 : 3 1 20 5 1 ; + assert.false version-less 3 1 10 : 3 1 10 0 0 ; + assert.false version-less 3 1 10 0 0 : 3 1 10 ; + assert.false version-less 3 1 10 0 : 3 1 10 0 0 ; + assert.false version-less 3 1 10 0 : 03 1 10 0 0 ; + assert.false version-less 03 1 10 0 : 3 1 10 0 0 ; + + # TODO: Add tests for invalid input data being sent to version-less. +} diff --git a/mosesdecoder/jam-files/boost-build/build/virtual-target.jam b/mosesdecoder/jam-files/boost-build/build/virtual-target.jam new file mode 100644 index 0000000000000000000000000000000000000000..2e8446bcc8d3327449533e2480d08ccfe8f56344 --- /dev/null +++ b/mosesdecoder/jam-files/boost-build/build/virtual-target.jam @@ -0,0 +1,1317 @@ +# Copyright 2003 Dave Abrahams +# Copyright 2005, 2006 Rene Rivera +# Copyright 2002, 2003, 2004, 2005, 2006 Vladimir Prus +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) + +# Implements virtual targets, which correspond to actual files created during a +# build, but are not yet targets in Jam sense. They are needed, for example, +# when searching for possible transformation sequences, when it is not yet known +# whether a particular target should be created at all. + +import "class" : new ; +import errors ; +import path ; +import sequence ; +import set ; +import type ; +import utility ; + + +# +--------------------------+ +# | virtual-target | +# +==========================+ +# | actualize | +# +--------------------------+ +# | actualize-action() = 0 | +# | actualize-location() = 0 | +# +----------------+---------+ +# | +# ^ +# / \ +# +-+-+ +# | +# +---------------------+ +-------+--------------+ +# | action | | abstract-file-target | +# +=====================| * +======================+ +# | action-name | +--+ action | +# | properties | | +----------------------+ +# +---------------------+--+ | actualize-action() | +# | actualize() |0..1 +-----------+----------+ +# | path() | | +# | adjust-properties() | sources | +# | actualize-sources() | targets | +# +------+--------------+ ^ +# | / \ +# ^ +-+-+ +# / \ | +# +-+-+ +-------------+-------------+ +# | | | +# | +------+---------------+ +--------+-------------+ +# | | file-target | | searched-lib-target | +# | +======================+ +======================+ +# | | actualize-location() | | actualize-location() | +# | +----------------------+ +----------------------+ +# | +# +-+------------------------------+ +# | | +# +----+----------------+ +---------+-----------+ +# | compile-action | | link-action | +# +=====================+ +=====================+ +# | adjust-properties() | | adjust-properties() | +# +---------------------+ | actualize-sources() | +# +---------------------+ +# +# The 'compile-action' and 'link-action' classes are not defined here but in +# builtin.jam modules. They are shown in the diagram to give the big picture. + + +# Models a potential target. It can be converted into a Jam target and used in +# building, if needed. However, it can be also dropped, which allows us to +# search for different transformations and select only one. +# +class virtual-target +{ + import scanner ; + import sequence ; + import utility ; + import virtual-target ; + + rule __init__ ( + name # Target/project name. + : project # Project to which this target belongs. + ) + { + self.name = $(name) ; + self.project = $(project) ; + self.dependencies = ; + } + + # Name of this target. + # + rule name ( ) + { + return $(self.name) ; + } + + # Project of this target. + # + rule project ( ) + { + return $(self.project) ; + } + + # Adds additional 'virtual-target' instances this one depends on. + # + rule depends ( d + ) + { + self.dependencies = [ sequence.merge $(self.dependencies) : + [ sequence.insertion-sort $(d) ] ] ; + } + + rule dependencies ( ) + { + return $(self.dependencies) ; + } + + rule always ( ) + { + .always = 1 ; + } + + # Generates all the actual targets and sets up build actions for this + # target. + # + # If 'scanner' is specified, creates an additional target with the same + # location as the actual target, which will depend on the actual target and + # be associated with a 'scanner'. That additional target is returned. See + # the docs (#dependency_scanning) for rationale. Target must correspond to a + # file if 'scanner' is specified. + # + # If scanner is not specified then the actual target is returned. + # + rule actualize ( scanner ? ) + { + local actual-name = [ actualize-no-scanner ] ; + + if $(.always) + { + ALWAYS $(actual-name) ; + } + + if ! $(scanner) + { + return $(actual-name) ; + } + else + { + # Add the scanner instance to the grist for name. + local g = [ sequence.join + [ utility.ungrist $(actual-name:G) ] $(scanner) : - ] ; + local name = $(actual-name:G=$(g)) ; + + if ! $(self.made.$(name)) + { + self.made.$(name) = true ; + + DEPENDS $(name) : $(actual-name) ; + + actualize-location $(name) ; + + scanner.install $(scanner) : $(name) $(__name__) ; + } + return $(name) ; + } + } + +# private: (overridables) + + # Sets up build actions for 'target'. Should call appropriate rules and set + # target variables. + # + rule actualize-action ( target ) + { + errors.error "method should be defined in derived classes" ; + } + + # Sets up variables on 'target' which specify its location. + # + rule actualize-location ( target ) + { + errors.error "method should be defined in derived classes" ; + } + + # If the target is a generated one, returns the path where it will be + # generated. Otherwise, returns an empty list. + # + rule path ( ) + { + errors.error "method should be defined in derived classes" ; + } + + # Returns the actual target name to be used in case when no scanner is + # involved. + # + rule actual-name ( ) + { + errors.error "method should be defined in derived classes" ; + } + +# implementation + rule actualize-no-scanner ( ) + { + # In fact, we just need to merge virtual-target with + # abstract-file-target as the latter is the only class derived from the + # former. But that has been left for later. + + errors.error "method should be defined in derived classes" ; + } +} + + +# Target corresponding to a file. The exact mapping for file is not yet +# specified in this class. (TODO: Actually, the class name could be better...) +# +# May be a source file (when no action is specified) or a derived file +# (otherwise). +# +# The target's grist is a concatenation of its project's location, action +# properties (for derived targets) and, optionally, value identifying the main +# target. +# +class abstract-file-target : virtual-target +{ + import project ; + import regex ; + import sequence ; + import path ; + import type ; + import property-set ; + import indirect ; + + rule __init__ ( + name # Target's name. + exact ? # If non-empty, the name is exactly the name created file + # should have. Otherwise, the '__init__' method will add a + # suffix obtained from 'type' by calling + # 'type.generated-target-suffix'. + : type ? # Target's type. + : project + : action ? + ) + { + virtual-target.__init__ $(name) : $(project) ; + + self.type = $(type) ; + self.action = $(action) ; + if $(action) + { + $(action).add-targets $(__name__) ; + + if $(self.type) && ! $(exact) + { + _adjust-name $(name) ; + } + } + } + + rule type ( ) + { + return $(self.type) ; + } + + # Sets the path. When generating target name, it will override any path + # computation from properties. + # + rule set-path ( path ) + { + self.path = [ path.native $(path) ] ; + } + + # Returns the currently set action. + # + rule action ( ) + { + return $(self.action) ; + } + + # Sets/gets the 'root' flag. Target is root if it directly corresponds to + # some variant of a main target. + # + rule root ( set ? ) + { + if $(set) + { + self.root = true ; + } + return $(self.root) ; + } + + # Gets or sets the subvariant which created this target. Subvariant is set + # when target is brought into existance and is never changed after that. In + # particular, if a target is shared by a subvariant, only the first is + # stored. + # + rule creating-subvariant ( s ? # If specified, specifies the value to set, + # which should be a 'subvariant' class + # instance. + ) + { + if $(s) && ! $(self.creating-subvariant) + { + self.creating-subvariant = $(s) ; + } + return $(self.creating-subvariant) ; + } + + rule actualize-action ( target ) + { + if $(self.action) + { + $(self.action).actualize ; + } + } + + # Return a human-readable representation of this target. If this target has + # an action, that is: + # + # { -. ... } + # + # otherwise, it is: + # + # { . } + # + rule str ( ) + { + local action = [ action ] ; + local name-dot-type = [ sequence.join $(self.name) "." $(self.type) ] ; + + if $(action) + { + local sources = [ $(action).sources ] ; + local action-name = [ $(action).action-name ] ; + + local ss ; + for local s in $(sources) + { + ss += [ $(s).str ] ; + } + + return "{" $(action-name)-$(name-dot-type) $(ss) "}" ; + } + else + { + return "{" $(name-dot-type) "}" ; + } + } + + rule less ( a ) + { + if [ str ] < [ $(a).str ] + { + return true ; + } + } + + rule equal ( a ) + { + if [ str ] = [ $(a).str ] + { + return true ; + } + } + +# private: + rule actual-name ( ) + { + if ! $(self.actual-name) + { + local grist = [ grist ] ; + local basename = [ path.native $(self.name) ] ; + self.actual-name = <$(grist)>$(basename) ; + } + return $(self.actual-name) ; + } + + # Helper to 'actual-name', above. Computes a unique prefix used to + # distinguish this target from other targets with the same name creating + # different files. + # + rule grist ( ) + { + # Depending on target, there may be different approaches to generating + # unique prefixes. We generate prefixes in the form: + # + local path = [ path ] ; + if $(path) + { + # The target will be generated to a known path. Just use the path + # for identification, since path is as unique as it can get. + return p$(path) ; + } + else + { + # File is either source, which will be searched for, or is not a + # file at all. Use the location of project for distinguishing. + local project-location = [ $(self.project).get location ] ; + local location-grist = [ sequence.join [ regex.split + $(project-location) "/" ] : "!" ] ; + + if $(self.action) + { + local ps = [ $(self.action).properties ] ; + local property-grist = [ $(ps).as-path ] ; + # 'property-grist' can be empty when 'ps' is an empty property + # set. + if $(property-grist) + { + location-grist = $(location-grist)/$(property-grist) ; + } + } + + return l$(location-grist) ; + } + } + + # Given the target name specified in constructor, returns the name which + # should be really used, by looking at the properties. Tag properties + # need to be specified as @rule-name. This makes Boost Build call the + # specified rule with the target name, type and properties to get the new + # name. If no property is specified or the rule specified by + # returns nothing, returns the result of calling + # virtual-target.add-prefix-and-suffix. + # + rule _adjust-name ( specified-name ) + { + local ps ; + if $(self.action) + { + ps = [ $(self.action).properties ] ; + } + else + { + ps = [ property-set.empty ] ; + } + + # We add ourselves to the properties so that any tag rule can get more + # direct information about the target than just that available through + # the properties. This is useful in implementing name changes based on + # the sources of the target. For example to make unique names of object + # files based on the source file. --grafik + ps = [ property-set.create [ $(ps).raw ] $(__name__) ] ; + + local tag = [ $(ps).get ] ; + + if $(tag) + { + local rule-name = [ MATCH ^@(.*) : $(tag) ] ; + if $(rule-name) + { + if $(tag[2]) + { + errors.error "@rulename is present but is not the only" + " feature" ; + } + + self.name = [ indirect.call $(rule-name) $(specified-name) + : $(self.type) : $(ps) ] ; + } + else + { + errors.error + "The value of the feature must be '@rule-name'" ; + } + } + + # If there is no tag or the tag rule returned nothing. + if ! $(tag) || ! $(self.name) + { + self.name = [ virtual-target.add-prefix-and-suffix $(specified-name) + : $(self.type) : $(ps) ] ; + } + } + + rule actualize-no-scanner ( ) + { + local name = [ actual-name ] ; + + # Do anything only on the first invocation. + if ! $(self.made.$(name)) + { + self.made.$(name) = true ; + + if $(self.action) + { + # For non-derived target, we do not care if there are several + # virtual targets that refer to the same name. One case when + # this is unavoidable is when the file name is main.cpp and two + # targets have types CPP (for compiling) and MOCCABLE_CPP (for + # conversion to H via Qt tools). + virtual-target.register-actual-name $(name) : $(__name__) ; + } + + for local i in $(self.dependencies) + { + DEPENDS $(name) : [ $(i).actualize ] ; + } + + actualize-location $(name) ; + actualize-action $(name) ; + } + return $(name) ; + } +} + + +# Appends the suffix appropriate to 'type/property-set' combination to the +# specified name and returns the result. +# +rule add-prefix-and-suffix ( specified-name : type ? : property-set ) +{ + local suffix = [ type.generated-target-suffix $(type) : $(property-set) ] ; + + # Handle suffixes for which no leading dot is desired. Those are specified + # by enclosing them in <...>. Needed by python so it can create "_d.so" + # extensions, for example. + if $(suffix:G) + { + suffix = [ utility.ungrist $(suffix) ] ; + } + else + { + suffix = .$(suffix) ; + } + + local prefix = [ type.generated-target-prefix $(type) : $(property-set) ] ; + + if [ MATCH ^($(prefix)) : $(specified-name) ] + { + prefix = ; + } + return $(prefix:E="")$(specified-name)$(suffix:E="") ; +} + + +# File targets with explicitly known location. +# +# The file path is determined as +# * Value passed to the 'set-path' method, if any. +# * For derived files, project's build dir, joined with components that +# describe action properties. If free properties are not equal to the +# project's reference properties an element with the name of the main +# target is added. +# * For source files, project's source dir. +# +# The file suffix is determined as: +# * The value passed to the 'suffix' method, if any. +# * The suffix corresponding to the target's type. +# +class file-target : abstract-file-target +{ + import "class" : new ; + import common ; + import errors ; + + rule __init__ ( + name exact ? + : type ? # Optional type for this target. + : project + : action ? + : path ? + ) + { + abstract-file-target.__init__ $(name) $(exact) : $(type) : $(project) : + $(action) ; + + self.path = $(path) ; + } + + rule clone-with-different-type ( new-type ) + { + return [ new file-target $(self.name) exact : $(new-type) : + $(self.project) : $(self.action) : $(self.path) ] ; + } + + rule actualize-location ( target ) + { + if $(self.action) + { + # This is a derived file. + local path = [ path ] ; + LOCATE on $(target) = $(path) ; + + # Make sure the path exists. + DEPENDS $(target) : $(path) ; + common.MkDir $(path) ; + + # It is possible that the target name includes a directory too, for + # example when installing headers. Create that directory. + if $(target:D) + { + local d = $(target:D) ; + d = $(d:R=$(path)) ; + DEPENDS $(target) : $(d) ; + common.MkDir $(d) ; + } + + # For a real file target, we create a fake target depending on the + # real target. This allows us to run + # + # bjam hello.o + # + # without trying to guess the name of the real target. Note that the + # target has no directory name and uses a special grist. + # + # First, that means that "bjam hello.o" will build all known hello.o + # targets. Second, the grist makes sure this target will not be + # confused with other targets, for example, if we have subdir 'test' + # with target 'test' in it that includes a 'test.o' file, then the + # target for directory will be just 'test' the target for test.o + # will be test.o and the target we create below + # will be test.o + DEPENDS $(target:G=e) : $(target) ; + # Allow bjam / to work. This will not catch all + # possible ways to refer to the path (relative/absolute, extra ".", + # various "..", but should help in obvious cases. + DEPENDS $(target:G=e:R=$(path)) : $(target) ; + } + else + { + SEARCH on $(target) = [ path.native $(self.path) ] ; + } + } + + # Returns the directory for this target. + # + rule path ( ) + { + if ! $(self.path) + { + if $(self.action) + { + local p = [ $(self.action).properties ] ; + local path,relative-to-build-dir = [ $(p).target-path ] ; + local path = $(path,relative-to-build-dir[1]) ; + local relative-to-build-dir = $(path,relative-to-build-dir[2]) ; + + if $(relative-to-build-dir) + { + path = [ path.join [ $(self.project).build-dir ] $(path) ] ; + } + + self.path = [ path.native $(path) ] ; + } + } + return $(self.path) ; + } +} + + +class notfile-target : abstract-file-target +{ + rule __init__ ( name : project : action ? ) + { + abstract-file-target.__init__ $(name) : : $(project) : $(action) ; + } + + # Returns nothing to indicate that the target's path is not known. + # + rule path ( ) + { + return ; + } + + rule actualize-location ( target ) + { + NOTFILE $(target) ; + ALWAYS $(target) ; + # TEMPORARY $(target) ; + NOUPDATE $(target) ; + } +} + + +# Class representing an action. Both 'targets' and 'sources' should list +# instances of 'virtual-target'. Action name should name a rule with this +# prototype: +# rule action-name ( targets + : sources * : properties * ) +# Targets and sources are passed as actual Jam targets. The rule may not +# establish additional dependency relationships. +# +class action +{ + import "class" ; + import errors ; + import type ; + import toolset ; + import property-set ; + import indirect ; + import path ; + import set : difference ; + + rule __init__ ( sources * : action-name + : property-set ? ) + { + self.sources = $(sources) ; + + self.action-name = [ indirect.make-qualified $(action-name) ] ; + + if ! $(property-set) + { + property-set = [ property-set.empty ] ; + } + + if ! [ class.is-instance $(property-set) ] + { + errors.error "Property set instance required" ; + } + + self.properties = $(property-set) ; + } + + rule add-targets ( targets * ) + { + self.targets += $(targets) ; + } + + rule replace-targets ( old-targets * : new-targets * ) + { + self.targets = [ set.difference $(self.targets) : $(old-targets) ] ; + self.targets += $(new-targets) ; + } + + rule targets ( ) + { + return $(self.targets) ; + } + + rule sources ( ) + { + return $(self.sources) ; + } + + rule action-name ( ) + { + return $(self.action-name) ; + } + + rule properties ( ) + { + return $(self.properties) ; + } + + # Generates actual build instructions. + # + rule actualize ( ) + { + if ! $(self.actualized) + { + self.actualized = true ; + + local ps = [ properties ] ; + local properties = [ adjust-properties $(ps) ] ; + + local actual-targets ; + for local i in [ targets ] + { + actual-targets += [ $(i).actualize ] ; + } + + actualize-sources [ sources ] : $(properties) ; + + DEPENDS $(actual-targets) : $(self.actual-sources) + $(self.dependency-only-sources) ; + + # This works around a bug with -j and actions that + # produce multiple target, where: + # - dependency on the first output is found, and + # the action is started + # - dependency on the second output is found, and + # bjam noticed that command is already running + # - instead of waiting for the command, dependents + # of the second targets are immediately updated. + if $(actual-targets[2]) + { + INCLUDES $(actual-targets) : $(actual-targets) ; + } + + # Action name can include additional argument to rule, which should + # not be passed to 'set-target-variables' + toolset.set-target-variables + [ indirect.get-rule $(self.action-name[1]) ] $(actual-targets) + : $(properties) ; + + # Reflect ourselves in a variable for the target. This allows + # looking up additional info for the action given the raw target. + # For example to debug or output action information from action + # rules. + .action on $(actual-targets) = $(__name__) ; + + indirect.call $(self.action-name) $(actual-targets) + : $(self.actual-sources) : [ $(properties).raw ] ; + + # Since we set up the creating action here, we set up the action for + # cleaning up as well. + common.Clean clean-all : $(actual-targets) ; + } + } + + # Helper for 'actualize-sources'. For each passed source, actualizes it with + # the appropriate scanner. Returns the actualized virtual targets. + # + rule actualize-source-type ( sources * : property-set ) + { + local result = ; + for local i in $(sources) + { + local scanner ; + if [ $(i).type ] + { + scanner = [ type.get-scanner [ $(i).type ] : $(property-set) ] ; + } + result += [ $(i).actualize $(scanner) ] ; + } + return $(result) ; + } + + # Creates actual Jam targets for sources. Initializes the following member + # variables: + # 'self.actual-sources' -- sources passed to the updating action. + # 'self.dependency-only-sources' -- sources marked as dependencies, but + # are not used otherwise. + # + # New values will be *appended* to the variables. They may be non-empty if + # caller wants it. + # + rule actualize-sources ( sources * : property-set ) + { + local dependencies = [ $(self.properties).get ] ; + + self.dependency-only-sources += + [ actualize-source-type $(dependencies) : $(property-set) ] ; + self.actual-sources += + [ actualize-source-type $(sources) : $(property-set) ] ; + + # This is used to help bjam find dependencies in generated headers and + # other main targets, e.g. in: + # + # make a.h : ....... ; + # exe hello : hello.cpp : a.h ; + # + # For bjam to find the dependency the generated target must be + # actualized (i.e. have its Jam target constructed). In the above case, + # if we are building just hello ("bjam hello"), 'a.h' will not be + # actualized unless we do it here. + local implicit = [ $(self.properties).get ] ; + for local i in $(implicit) + { + $(i:G=).actualize ; + } + } + + # Determines real properties when trying to build with 'properties'. This is + # the last chance to fix properties, for example to adjust includes to get + # generated headers correctly. Default implementation simply returns its + # argument. + # + rule adjust-properties ( property-set ) + { + return $(property-set) ; + } +} + + +# Action class which does nothing --- it produces the targets with specific +# properties out of nowhere. It is needed to distinguish virtual targets with +# different properties that are known to exist and have no actions which create +# them. +# +class null-action : action +{ + rule __init__ ( property-set ? ) + { + action.__init__ : .no-action : $(property-set) ; + } + + rule actualize ( ) + { + if ! $(self.actualized) + { + self.actualized = true ; + for local i in [ targets ] + { + $(i).actualize ; + } + } + } +} + + +# Class which acts exactly like 'action', except that its sources are not +# scanned for dependencies. +# +class non-scanning-action : action +{ + rule __init__ ( sources * : action-name + : property-set ? ) + { + action.__init__ $(sources) : $(action-name) : $(property-set) ; + } + + rule actualize-source-type ( sources * : property-set ) + { + local result ; + for local i in $(sources) + { + result += [ $(i).actualize ] ; + } + return $(result) ; + } +} + + +# Creates a virtual target with an appropriate name and type from 'file'. If a +# target with that name in that project already exists, returns that already +# created target. +# +# FIXME: a more correct way would be to compute the path to the file, based on +# name and source location for the project, and use that path to determine if +# the target has already been created. This logic should be shared with how we +# usually find targets identified by a specific target id. It should also be +# updated to work correctly when the file is specified using both relative and +# absolute paths. +# +# TODO: passing a project with all virtual targets is starting to be annoying. +# +rule from-file ( file : file-loc : project ) +{ + import type ; # Had to do this here to break a circular dependency. + + # Check whether we already created a target corresponding to this file. + local path = [ path.root [ path.root $(file) $(file-loc) ] [ path.pwd ] ] ; + + if $(.files.$(path)) + { + return $(.files.$(path)) ; + } + else + { + local name = [ path.make $(file) ] ; + local type = [ type.type $(file) ] ; + local result ; + + result = [ new file-target $(file) : $(type) : $(project) : : + $(file-loc) ] ; + + .files.$(path) = $(result) ; + return $(result) ; + } +} + + +# Registers a new virtual target. Checks if there is already a registered target +# with the same name, type, project and subvariant properties as well as the +# same sources and equal action. If such target is found it is returned and a +# new 'target' is not registered. Otherwise, 'target' is registered and +# returned. +# +rule register ( target ) +{ + local signature = [ sequence.join + [ $(target).path ] [ $(target).name ] : - ] ; + + local result ; + for local t in $(.cache.$(signature)) + { + local a1 = [ $(t).action ] ; + local a2 = [ $(target).action ] ; + + if ! $(result) + { + if ! $(a1) && ! $(a2) + { + result = $(t) ; + } + else + { + if $(a1) && $(a2) && + ( [ $(a1).action-name ] = [ $(a2).action-name ] ) && + ( [ $(a1).sources ] = [ $(a2).sources ] ) + { + local ps1 = [ $(a1).properties ] ; + local ps2 = [ $(a2).properties ] ; + local p1 = [ $(ps1).base ] [ $(ps1).free ] [ set.difference + [ $(ps1).dependency ] : [ $(ps1).incidental ] ] ; + local p2 = [ $(ps2).base ] [ $(ps2).free ] [ set.difference + [ $(ps2).dependency ] : [ $(ps2).incidental ] ] ; + if $(p1) = $(p2) + { + result = $(t) ; + } + } + } + } + } + + if ! $(result) + { + .cache.$(signature) += $(target) ; + result = $(target) ; + } + + .recent-targets += $(result) ; + .all-targets += $(result) ; + + return $(result) ; +} + + +# Each target returned by 'register' is added to the .recent-targets list, +# returned by this function. This allows us to find all virtual targets created +# when building a specific main target, even those constructed only as +# intermediate targets. +# +rule recent-targets ( ) +{ + return $(.recent-targets) ; +} + + +rule clear-recent-targets ( ) +{ + .recent-targets = ; +} + + +# Returns all virtual targets ever created. +# +rule all-targets ( ) +{ + return $(.all-targets) ; +} + + +# Returns all targets from 'targets' with types equal to 'type' or derived from +# it. +# +rule select-by-type ( type : targets * ) +{ + local result ; + for local t in $(targets) + { + if [ type.is-subtype [ $(t).type ] $(type) ] + { + result += $(t) ; + } + } + return $(result) ; +} + + +rule register-actual-name ( actual-name : virtual-target ) +{ + if $(.actual.$(actual-name)) + { + local cs1 = [ $(.actual.$(actual-name)).creating-subvariant ] ; + local cs2 = [ $(virtual-target).creating-subvariant ] ; + local cmt1 = [ $(cs1).main-target ] ; + local cmt2 = [ $(cs2).main-target ] ; + + local action1 = [ $(.actual.$(actual-name)).action ] ; + local action2 = [ $(virtual-target).action ] ; + local properties-added ; + local properties-removed ; + if $(action1) && $(action2) + { + local p1 = [ $(action1).properties ] ; + p1 = [ $(p1).raw ] ; + local p2 = [ $(action2).properties ] ; + p2 = [ $(p2).raw ] ; + properties-removed = [ set.difference $(p1) : $(p2) ] ; + properties-removed ?= "none" ; + properties-added = [ set.difference $(p2) : $(p1) ] ; + properties-added ?= "none" ; + } + errors.error "Duplicate name of actual target:" $(actual-name) + : "previous virtual target" [ $(.actual.$(actual-name)).str ] + : "created from" [ $(cmt1).full-name ] + : "another virtual target" [ $(virtual-target).str ] + : "created from" [ $(cmt2).full-name ] + : "added properties:" $(properties-added) + : "removed properties:" $(properties-removed) ; + } + else + { + .actual.$(actual-name) = $(virtual-target) ; + } +} + + +# Traverses the dependency graph of 'target' and return all targets that will be +# created before this one is created. If the root of some dependency graph is +# found during traversal, it is either included or not, depending on the +# 'include-roots' value. In either case traversal stops at root targets, i.e. +# root target sources are not traversed. +# +rule traverse ( target : include-roots ? : include-sources ? ) +{ + local result ; + if [ $(target).action ] + { + local action = [ $(target).action ] ; + # This includes the 'target' as well. + result += [ $(action).targets ] ; + + for local t in [ $(action).sources ] + { + if ! [ $(t).root ] + { + result += [ traverse $(t) : $(include-roots) : $(include-sources) ] ; + } + else if $(include-roots) + { + result += $(t) ; + } + } + } + else if $(include-sources) + { + result = $(target) ; + } + return $(result) ; +} + + +# Takes an 'action' instance and creates a new instance of it and all targets +# produced by the action. The rule-name and properties are set to +# 'new-rule-name' and 'new-properties', if those are specified. Returns the +# cloned action. +# +rule clone-action ( action : new-project : new-action-name ? : new-properties ? ) +{ + if ! $(new-action-name) + { + new-action-name = [ $(action).action-name ] ; + } + if ! $(new-properties) + { + new-properties = [ $(action).properties ] ; + } + + local action-class = [ modules.peek $(action) : __class__ ] ; + local cloned-action = [ class.new $(action-class) + [ $(action).sources ] : $(new-action-name) : $(new-properties) ] ; + + local cloned-targets ; + for local target in [ $(action).targets ] + { + local n = [ $(target).name ] ; + # Do not modify produced target names. + local cloned-target = [ class.new file-target $(n) exact : + [ $(target).type ] : $(new-project) : $(cloned-action) ] ; + local d = [ $(target).dependencies ] ; + if $(d) + { + $(cloned-target).depends $(d) ; + } + $(cloned-target).root [ $(target).root ] ; + $(cloned-target).creating-subvariant [ $(target).creating-subvariant ] ; + + cloned-targets += $(cloned-target) ; + } + + return $(cloned-action) ; +} + + +class subvariant +{ + import sequence ; + import type ; + + rule __init__ ( main-target # The instance of main-target class. + : property-set # Properties requested for this target. + : sources * + : build-properties # Actually used properties. + : sources-usage-requirements # Properties propagated from sources. + : created-targets * ) # Top-level created targets. + { + self.main-target = $(main-target) ; + self.properties = $(property-set) ; + self.sources = $(sources) ; + self.build-properties = $(build-properties) ; + self.sources-usage-requirements = $(sources-usage-requirements) ; + self.created-targets = $(created-targets) ; + + # Pre-compose a list of other dependency graphs this one depends on. + local deps = [ $(build-properties).get ] ; + for local d in $(deps) + { + self.other-dg += [ $(d:G=).creating-subvariant ] ; + } + + self.other-dg = [ sequence.unique $(self.other-dg) ] ; + } + + rule main-target ( ) + { + return $(self.main-target) ; + } + + rule created-targets ( ) + { + return $(self.created-targets) ; + } + + rule requested-properties ( ) + { + return $(self.properties) ; + } + + rule build-properties ( ) + { + return $(self.build-properties) ; + } + + rule sources-usage-requirements ( ) + { + return $(self.sources-usage-requirements) ; + } + + rule set-usage-requirements ( usage-requirements ) + { + self.usage-requirements = $(usage-requirements) ; + } + + rule usage-requirements ( ) + { + return $(self.usage-requirements) ; + } + + # Returns all targets referenced by this subvariant, either directly or + # indirectly, and either as sources, or as dependency properties. Targets + # referred to using the dependency property are returned as properties, not + # targets. + # + rule all-referenced-targets ( theset ) + { + # Find directly referenced targets. + local deps = [ $(self.build-properties).dependency ] ; + local all-targets = $(self.sources) $(deps) ; + + # Find other subvariants. + local r ; + for local t in $(all-targets) + { + if ! [ $(theset).contains $(t) ] + { + $(theset).add $(t) ; + r += [ $(t:G=).creating-subvariant ] ; + } + } + r = [ sequence.unique $(r) ] ; + for local s in $(r) + { + if $(s) != $(__name__) + { + $(s).all-referenced-targets $(theset) ; + } + } + } + + # Returns the properties specifying implicit include paths to generated + # headers. This traverses all targets in this subvariant and subvariants + # referred by properties. For all targets of type + # 'target-type' (or for all targets, if 'target-type' is not specified), the + # result will contain <$(feature)>path-to-that-target. + # + rule implicit-includes ( feature : target-type ? ) + { + local key = ii$(feature)-$(target-type:E="") ; + if ! $($(key))-is-not-empty + { + local target-paths = [ all-target-directories $(target-type) ] ; + target-paths = [ sequence.unique $(target-paths) ] ; + local result = $(target-paths:G=$(feature)) ; + if ! $(result) + { + result = "" ; + } + $(key) = $(result) ; + } + if $($(key)) = "" + { + return ; + } + else + { + return $($(key)) ; + } + } + + rule all-target-directories ( target-type ? ) + { + if ! $(self.target-directories) + { + compute-target-directories $(target-type) ; + } + return $(self.target-directories) ; + } + + rule compute-target-directories ( target-type ? ) + { + local result ; + for local t in $(self.created-targets) + { + # Skip targets of the wrong type. + if ! $(target-type) || + [ type.is-derived [ $(t).type ] $(target-type) ] + { + result = [ sequence.merge $(result) : [ $(t).path ] ] ; + } + } + for local d in $(self.other-dg) + { + result += [ $(d).all-target-directories $(target-type) ] ; + } + self.target-directories = $(result) ; + } +} diff --git a/mosesdecoder/jam-files/boost-build/util/assert.jam b/mosesdecoder/jam-files/boost-build/util/assert.jam new file mode 100644 index 0000000000000000000000000000000000000000..abedad525926129777017cdb5386206e029e9add --- /dev/null +++ b/mosesdecoder/jam-files/boost-build/util/assert.jam @@ -0,0 +1,336 @@ +# Copyright 2001, 2002, 2003 Dave Abrahams +# Copyright 2006 Rene Rivera +# Copyright 2002, 2003 Vladimir Prus +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) + +import errors ; +import modules ; + + +################################################################################ +# +# Private implementation details. +# +################################################################################ + +# Rule added as a replacement for the regular Jam = operator but which does not +# ignore trailing empty string elements. +# +local rule exact-equal-test ( lhs * : rhs * ) +{ + local lhs_extended = $(lhs) xxx ; + local rhs_extended = $(rhs) xxx ; + if $(lhs_extended) = $(rhs_extended) + { + return true ; + } +} + + +# Two lists are considered set-equal if they contain the same elements, ignoring +# duplicates and ordering. +# +local rule set-equal-test ( set1 * : set2 * ) +{ + if ( $(set1) in $(set2) ) && ( $(set2) in $(set1) ) + { + return true ; + } +} + + +################################################################################ +# +# Public interface. +# +################################################################################ + +# Assert the equality of A and B, ignoring trailing empty string elements. +# +rule equal ( a * : b * ) +{ + if $(a) != $(b) + { + errors.error-skip-frames 3 assertion failure: \"$(a)\" "==" \"$(b)\" + (ignoring trailing empty strings) ; + } +} + + +# Assert that the result of calling RULE-NAME on the given arguments has a false +# logical value (is either an empty list or all empty strings). +# +rule false ( rule-name args * : * ) +{ + local result ; + module [ CALLER_MODULE ] + { + modules.poke assert : result : [ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) + : $(7) : $(8) : $(9) ] ; + } + + if $(result) + { + errors.error-skip-frames 3 assertion failure: Expected false result from + "[" $(rule-name) [ errors.lol->list $(args) : $(2) : $(3) : $(4) : + $(5) : $(6) : $(7) : $(8) : $(9) ] "]" : Got: "[" \"$(result)\" "]" ; + } +} + + +# Assert that ELEMENT is present in LIST. +# +rule "in" ( element : list * ) +{ + if ! $(element) in $(list) + { + errors.error-skip-frames 3 assertion failure: Expected \"$(element)\" in + "[" \"$(list)\" "]" ; + } +} + + +# Assert the inequality of A and B, ignoring trailing empty string elements. +# +rule not-equal ( a * : b * ) +{ + if $(a) = $(b) + { + errors.error-skip-frames 3 assertion failure: \"$(a)\" "!=" \"$(b)\" + (ignoring trailing empty strings) ; + } +} + + +# Assert that ELEMENT is not present in LIST. +# +rule not-in ( element : list * ) +{ + if $(element) in $(list) + { + errors.error-skip-frames 3 assertion failure: Did not expect + \"$(element)\" in "[" \"$(list)\" "]" ; + } +} + + +# Assert the inequality of A and B as sets. +# +rule not-set-equal ( a * : b * ) +{ + if [ set-equal-test $(a) : $(b) ] + { + errors.error-skip-frames 3 assertion failure: Expected "[" \"$(a)\" "]" + and "[" \"$(b)\" "]" to not be equal as sets ; + } +} + + +# Assert that A and B are not exactly equal, not ignoring trailing empty string +# elements. +# +rule not-exact-equal ( a * : b * ) +{ + if [ exact-equal-test $(a) : $(b) ] + { + errors.error-skip-frames 3 assertion failure: \"$(a)\" "!=" \"$(b)\" ; + } +} + + +# Assert that EXPECTED is the result of calling RULE-NAME with the given +# arguments. +# +rule result ( expected * : rule-name args * : * ) +{ + local result ; + module [ CALLER_MODULE ] + { + modules.poke assert : result : [ $(2) : $(3) : $(4) : $(5) : $(6) : $(7) + : $(8) : $(9) ] ; + } + + if ! [ exact-equal-test $(result) : $(expected) ] + { + errors.error-skip-frames 3 assertion failure: "[" $(rule-name) [ + errors.lol->list $(args) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : + $(9) ] "]" : Expected: "[" \"$(expected)\" "]" : Got: "[" + \"$(result)\" "]" ; + } +} + + +# Assert that EXPECTED is set-equal (i.e. duplicates and ordering are ignored) +# to the result of calling RULE-NAME with the given arguments. Note that rules +# called this way may accept at most 8 parameters. +# +rule result-set-equal ( expected * : rule-name args * : * ) +{ + local result ; + module [ CALLER_MODULE ] + { + modules.poke assert : result : [ $(2) : $(3) : $(4) : $(5) : $(6) : $(7) + : $(8) : $(9) ] ; + } + + if ! [ set-equal-test $(result) : $(expected) ] + { + errors.error-skip-frames 3 assertion failure: "[" $(rule-name) [ + errors.lol->list $(args) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : + $(9) ] "]" : Expected: "[" \"$(expected)\" "]" : Got: "[" + \"$(result)\" "]" ; + } +} + + +# Assert the equality of A and B as sets. +# +rule set-equal ( a * : b * ) +{ + if ! [ set-equal-test $(a) : $(b) ] + { + errors.error-skip-frames 3 assertion failure: Expected "[" \"$(a)\" "]" + and "[" \"$(b)\" "]" to be equal as sets ; + } +} + + +# Assert that the result of calling RULE-NAME on the given arguments has a true +# logical value (is neither an empty list nor all empty strings). +# +rule true ( rule-name args * : * ) +{ + local result ; + module [ CALLER_MODULE ] + { + modules.poke assert : result : [ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) + : $(7) : $(8) : $(9) ] ; + } + + if ! $(result) + { + errors.error-skip-frames 3 assertion failure: Expected true result from + "[" $(rule-name) [ errors.lol->list $(args) : $(2) : $(3) : $(4) : + $(5) : $(6) : $(7) : $(8) : $(9) ] "]" ; + } +} + + +# Assert the exact equality of A and B, not ignoring trailing empty string +# elements. +# +rule exact-equal ( a * : b * ) +{ + if ! [ exact-equal-test $(a) : $(b) ] + { + errors.error-skip-frames 3 assertion failure: \"$(a)\" "==" \"$(b)\" ; + } +} + + +# Assert that the given variable is not an empty list. +# +rule variable-not-empty ( name ) +{ + local value = [ modules.peek [ CALLER_MODULE ] : $(name) ] ; + if ! $(value)-is-not-empty + { + errors.error-skip-frames 3 assertion failure: Expected variable + \"$(name)\" not to be an empty list ; + } +} + + +rule __test__ ( ) +{ + # Helper rule used to avoid test duplication related to different list + # equality test rules. + # + local rule run-equality-test ( equality-assert : ignore-trailing-empty-strings ? ) + { + local not-equality-assert = not-$(equality-assert) ; + + # When the given equality test is expected to ignore trailing empty + # strings some of the test results should be inverted. + local not-equality-assert-i = not-$(equality-assert) ; + if $(ignore-trailing-empty-strings) + { + not-equality-assert-i = $(equality-assert) ; + } + + $(equality-assert) : ; + $(equality-assert) "" "" : "" "" ; + $(not-equality-assert-i) : "" "" ; + $(equality-assert) x : x ; + $(not-equality-assert) : x ; + $(not-equality-assert) "" : x ; + $(not-equality-assert) "" "" : x ; + $(not-equality-assert-i) x : x "" ; + $(equality-assert) x "" : x "" ; + $(not-equality-assert) x : "" x ; + $(equality-assert) "" x : "" x ; + + $(equality-assert) 1 2 3 : 1 2 3 ; + $(not-equality-assert) 1 2 3 : 3 2 1 ; + $(not-equality-assert) 1 2 3 : 1 5 3 ; + $(not-equality-assert) 1 2 3 : 1 "" 3 ; + $(not-equality-assert) 1 2 3 : 1 1 2 3 ; + $(not-equality-assert) 1 2 3 : 1 2 2 3 ; + $(not-equality-assert) 1 2 3 : 5 6 7 ; + + # Extra variables used here just to make sure Boost Jam or Boost Build + # do not handle lists with empty strings differently depending on + # whether they are literals or stored in variables. + + local empty = ; + local empty-strings = "" "" ; + local x-empty-strings = x "" "" ; + local empty-strings-x = "" "" x ; + + $(equality-assert) : $(empty) ; + $(not-equality-assert-i) "" : $(empty) ; + $(not-equality-assert-i) "" "" : $(empty) ; + $(not-equality-assert-i) : $(empty-strings) ; + $(not-equality-assert-i) "" : $(empty-strings) ; + $(equality-assert) "" "" : $(empty-strings) ; + $(equality-assert) $(empty) : $(empty) ; + $(equality-assert) $(empty-strings) : $(empty-strings) ; + $(not-equality-assert-i) $(empty) : $(empty-strings) ; + $(equality-assert) $(x-empty-strings) : $(x-empty-strings) ; + $(equality-assert) $(empty-strings-x) : $(empty-strings-x) ; + $(not-equality-assert) $(empty-strings-x) : $(x-empty-strings) ; + $(not-equality-assert-i) x : $(x-empty-strings) ; + $(not-equality-assert) x : $(empty-strings-x) ; + $(not-equality-assert-i) x : $(x-empty-strings) ; + $(not-equality-assert-i) x "" : $(x-empty-strings) ; + $(equality-assert) x "" "" : $(x-empty-strings) ; + $(not-equality-assert) x : $(empty-strings-x) ; + $(not-equality-assert) "" x : $(empty-strings-x) ; + $(equality-assert) "" "" x : $(empty-strings-x) ; + } + + + # --------------- + # Equality tests. + # --------------- + + run-equality-test equal : ignore-trailing-empty-strings ; + run-equality-test exact-equal ; + + + # ------------------------- + # assert.set-equal() tests. + # ------------------------- + + set-equal : ; + not-set-equal "" "" : ; + set-equal "" "" : "" ; + set-equal "" "" : "" "" ; + set-equal a b c : a b c ; + set-equal a b c : b c a ; + set-equal a b c a : a b c ; + set-equal a b c : a b c a ; + not-set-equal a b c : a b c d ; + not-set-equal a b c d : a b c ; +} diff --git a/mosesdecoder/jam-files/boost-build/util/doc.jam b/mosesdecoder/jam-files/boost-build/util/doc.jam new file mode 100644 index 0000000000000000000000000000000000000000..6e89d075ee2f67c236e6acf05b891588a6f2b8fb --- /dev/null +++ b/mosesdecoder/jam-files/boost-build/util/doc.jam @@ -0,0 +1,997 @@ +# Copyright 2002, 2005 Dave Abrahams +# Copyright 2002, 2003, 2006 Rene Rivera +# Copyright 2003 Vladimir Prus +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) + +# Documentation system, handles --help requests. +# It defines rules that attach documentation to modules, rules, and variables. +# Collects and generates documentation for the various parts of the build +# system. The documentation is collected from comments integrated into the code. + +import modules ; +import print ; +import set ; +import container ; +import "class" ; +import sequence ; +import path ; + + +# The type of output to generate. +# "console" is formated text echoed to the console (the default); +# "text" is formated text appended to the output file; +# "html" is HTML output to the file. +# +help-output = console ; + + +# The file to output documentation to when generating "text" or "html" help. +# This is without extension as the extension is determined by the type of +# output. +# +help-output-file = help ; + +# Whether to include local rules in help output. +# +.option.show-locals ?= ; + +# When showing documentation for a module, whether to also generate +# automatically the detailed docs for each item in the module. +# +.option.detailed ?= ; + +# Generate debug output as the help is generated and modules are parsed. +# +.option.debug ?= ; + +# Enable or disable a documentation option. +# +local rule set-option ( + option # The option name. + : value ? # Enabled (non-empty), or disabled (empty) +) +{ + .option.$(option) = $(value) ; +} + + +# Set the type of output. +# +local rule set-output ( type ) +{ + help-output = $(type) ; +} + + +# Set the output to a file. +# +local rule set-output-file ( file ) +{ + help-output-file = $(file) ; +} + + +# Extracts the brief comment from a complete comment. The brief comment is the +# first sentence. +# +local rule brief-comment ( + docs * # The comment documentation. +) +{ + local d = $(docs:J=" ") ; + local p = [ MATCH ".*([.])$" : $(d) ] ; + if ! $(p) { d = $(d)"." ; } + d = $(d)" " ; + local m = [ MATCH "^([^.]+[.])(.*)" : $(d) ] ; + local brief = $(m[1]) ; + while $(m[2]) && [ MATCH "^([^ ])" : $(m[2]) ] + { + m = [ MATCH "^([^.]+[.])(.*)" : $(m[2]) ] ; + brief += $(m[1]) ; + } + return $(brief:J="") ; +} + + +# Specifies the documentation for the current module. +# +local rule set-module-doc ( + module-name ? # The name of the module to document. + : docs * # The documentation for the module. +) +{ + module-name ?= * ; + + $(module-name).brief = [ brief-comment $(docs) ] ; + $(module-name).docs = $(docs) ; + + if ! $(module-name) in $(documented-modules) + { + documented-modules += $(module-name) ; + } +} + + +# Specifies the documentation for the current module. +# +local rule set-module-copyright ( + module-name ? # The name of the module to document. + : copyright * # The copyright for the module. +) +{ + module-name ?= * ; + + $(module-name).copy-brief = [ brief-comment $(copyright) ] ; + $(module-name).copy-docs = $(docs) ; + + if ! $(module-name) in $(documented-modules) + { + documented-modules += $(module-name) ; + } +} + + +# Specifies the documentation for a rule in the current module. If called in the +# global module, this documents a global rule. +# +local rule set-rule-doc ( + name # The name of the rule. + module-name ? # The name of the module to document. + is-local ? # Whether the rule is local to the module. + : docs * # The documentation for the rule. +) +{ + module-name ?= * ; + + $(module-name).$(name).brief = [ brief-comment $(docs) ] ; + $(module-name).$(name).docs = $(docs) ; + $(module-name).$(name).is-local = $(is-local) ; + + if ! $(name) in $($(module-name).rules) + { + $(module-name).rules += $(name) ; + } +} + + +# Specify a class, will turn a rule into a class. +# +local rule set-class-doc ( + name # The name of the class. + module-name ? # The name of the module to document. + : super-name ? # The super class name. +) +{ + module-name ?= * ; + + $(module-name).$(name).is-class = true ; + $(module-name).$(name).super-name = $(super-name) ; + $(module-name).$(name).class-rules = + [ MATCH "^($(name)[.].*)" : $($(module-name).rules) ] ; + $(module-name).$($(module-name).$(name).class-rules).is-class-rule = true ; + + $(module-name).classes += $(name) ; + $(module-name).class-rules += $($(module-name).$(name).class-rules) ; + $(module-name).rules = + [ set.difference $($(module-name).rules) : + $(name) $($(module-name).$(name).class-rules) ] ; +} + + +# Set the argument call signature of a rule. +# +local rule set-rule-arguments-signature ( + name # The name of the rule. + module-name ? # The name of the module to document. + : signature * # The arguments signature. +) +{ + module-name ?= * ; + + $(module-name).$(name).signature = $(signature) ; +} + + +# Specifies the documentation for an argument of a rule. +# +local rule set-argument-doc ( + name # The name of the argument. + qualifier # Argument syntax qualifier, "*", "+", etc. + rule-name # The name of the rule. + module-name ? # THe optional name of the module. + : docs * # The documentation. +) +{ + module-name ?= * ; + + $(module-name).$(rule-name).args.$(name).qualifier = $(qualifier) ; + $(module-name).$(rule-name).args.$(name).docs = $(docs) ; + + if ! $(name) in $($(module-name).$(rule-name).args) + { + $(module-name).$(rule-name).args += $(name) ; + } +} + + +# Specifies the documentation for a variable in the current module. If called in +# the global module, the global variable is documented. +# +local rule set-variable-doc ( + name # The name of the variable. + default # The default value. + initial # The initial value. + module-name ? # The name of the module to document. + : docs * # The documentation for the variable. +) +{ + module-name ?= * ; + + $(module-name).$(name).brief = [ brief-comment $(docs) ] ; + $(module-name).$(name).default = $(default) ; + $(module-name).$(name).initial = $(initial) ; + $(module-name).$(name).docs = $(docs) ; + + if ! $(name) in $($(module-name).variables) + { + $(module-name).variables += $(name) ; + } +} + + +# Generates a general description of the documentation and help system. +# +local rule print-help-top ( ) +{ + print.section "General command line usage" ; + + print.text " bjam [options] [properties] [targets] + + Options, properties and targets can be specified in any order. + " ; + + print.section "Important Options" ; + + print.list-start ; + print.list-item "--clean Remove targets instead of building" ; + print.list-item "-a Rebuild everything" ; + print.list-item "-n Don't execute the commands, only print them" ; + print.list-item "-d+2 Show commands as they are executed" ; + print.list-item "-d0 Supress all informational messages" ; + print.list-item "-q Stop at first error" ; + print.list-item "--debug-configuration Diagnose configuration" ; + print.list-item "--debug-building Report which targets are built with what properties" ; + print.list-item "--debug-generator Diagnose generator search/execution" ; + print.list-end ; + + print.section "Further Help" + The following options can be used to obtain additional documentation. + ; + + print.list-start ; + print.list-item "--help-options Print more obscure command line options." ; + print.list-item "--help-internal Boost.Build implementation details." ; + print.list-item "--help-doc-options Implementation details doc formatting." ; + print.list-end ; +} + + +# Generate Jam/Boost.Jam command usage information. +# +local rule print-help-usage ( ) +{ + print.section "Boost.Jam Usage" + "bjam [ options... ] targets..." + ; + print.list-start ; + print.list-item -a; + Build all targets, even if they are current. ; + print.list-item -fx; + Read '"x"' as the Jamfile for building instead of searching for the + Boost.Build system. ; + print.list-item -jx; + Run up to '"x"' commands concurrently. ; + print.list-item -n; + Do not execute build commands. Instead print out the commands as they + would be executed if building. ; + print.list-item -ox; + Output the used build commands to file '"x"'. ; + print.list-item -q; + Quit as soon as a build failure is encountered. Without this option + Boost.Jam will continue building as many targets as it can. + print.list-item -sx=y; + Sets a Jam variable '"x"' to the value '"y"', overriding any value that + variable would have from the environment. ; + print.list-item -tx; + Rebuild the target '"x"', even if it is up-to-date. ; + print.list-item -v; + Display the version of bjam. ; + print.list-item --x; + Any option not explicitly handled by Boost.Jam remains available to + build scripts using the '"ARGV"' variable. ; + print.list-item -dn; + Enables output of diagnostic messages. The debug level '"n"' and all + below it are enabled by this option. ; + print.list-item -d+n; + Enables output of diagnostic messages. Only the output for debug level + '"n"' is enabled. ; + print.list-end ; + print.section "Debug Levels" + Each debug level shows a different set of information. Usually with + higher levels producing more verbose information. The following levels + are supported: ; + print.list-start ; + print.list-item 0; + Turn off all diagnostic output. Only errors are reported. ; + print.list-item 1; + Show the actions taken for building targets, as they are executed. ; + print.list-item 2; + Show "quiet" actions and display all action text, as they are executed. ; + print.list-item 3; + Show dependency analysis, and target/source timestamps/paths. ; + print.list-item 4; + Show arguments of shell invocations. ; + print.list-item 5; + Show rule invocations and variable expansions. ; + print.list-item 6; + Show directory/header file/archive scans, and attempts at binding to targets. ; + print.list-item 7; + Show variable settings. ; + print.list-item 8; + Show variable fetches, variable expansions, and evaluation of '"if"' expressions. ; + print.list-item 9; + Show variable manipulation, scanner tokens, and memory usage. ; + print.list-item 10; + Show execution times for rules. ; + print.list-item 11; + Show parsing progress of Jamfiles. ; + print.list-item 12; + Show graph for target dependencies. ; + print.list-item 13; + Show changes in target status (fate). ; + print.list-end ; +} + + +# Generates description of options controlling the help system. This +# automatically reads the options as all variables in the doc module of the form +# ".option.*". +# +local rule print-help-options ( + module-name # The doc module. +) +{ + print.section "Help Options" + These are all the options available for enabling or disabling to control + the help system in various ways. Options can be enabled or disabled with + '"--help-enable-