tools/mosesdecoder/contrib/m4m/modules/eval-system.m4m

# -*- Makefile -*-

moses.threads ?= 4
moses.flags += -threads ${moses.threads}
moses.flags += -v 0 -t -text-type "test" -fd '${FACTORSEP}'

%.multi-bleu: | %.cleaned 
	$(lock)
	${multi-bleu} ${bleu.ref} < $*.cleaned > $@_ && mv $@_ $@
	$(unlock)

%.moses-out.wa: moses.flags += -alignment-output-file $*.output.wa
%.moses-out.wa: %.moses-out

# .SECONDEXPANSION:
moses.ini ?= 
%.moses-out: | ${moses.ini}
	echo MOSES $^ 
	$(lock)
	${moses} -i ${moses.input} -inputtype ${moses.inputtype} \
	-f ${moses.ini} ${moses.flags} > $@_ && mv $@_ $@
	$(unlock)

%.cleaned: %.moses-out
	$(lock)
	$(clean-decoder-output) < $< > $@_ && mv $@_ $@
	$(unlock)

%.natcased: %.cleaned
	$(eval $(call lock))
	$(detruecase) < $*.cleaned > $@_ && mv $@_ $@
	$(eval $(call unlock))


# $1: output base name
# $2: system to be evaluated
# $3: evaluation input

define bleu_eval

EVALUATIONS += $1
$1: moses.ini       := $2
$1: moses.input     := $3
$1: moses.inputtype := $(call guess-inputtype,$3)
$1: bleu.ref        := $$(shell echo $(patsubst %.${L1},%.${L2},$3) | perl -pe 's?/cfn[^/]+/?/cased/?')
$1.moses-out:  | $2 $3
$1.multi-bleu: | $(call reffiles,$3,$(dir $(patsubst %/,%,$(dir $3))))
$1: | $1.multi-bleu 

endef

EVALUATIONS :=

eval.sets ?= $(patsubst %.${L1}.gz,%,$(subst /raw/,/cased/,$(wildcard ${WDIR}/crp/tst/raw/*.${L1}.gz)))

ifdef tune.runs
define bleu_score_all_systems

$(foreach system,${SYSTEMS},\
$(foreach tuneset,${tune.sets},\
$(foreach evalset,${eval.sets},\
$(foreach run,$(shell seq ${tune.runs}),\
$(eval $(call bleu_eval,\
${system}/eval/$(notdir ${tuneset})/${run}/$(notdir ${evalset}),\
${system}/tuned/$(notdir ${tuneset})/${run}/moses.ini,\
${evalset}.${L1},${moses.inputtype.plaintext},${evalset}.${L2}))))))

endef
endif