File size: 3,653 Bytes
8c9ba62 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 | #!/usr/bin/env bash
# OPTIONAL. Score a finished run's checkpoints on the 1,308-task test split.
#
# bash scripts/eval.sh <method> [--with-base]
#
# Read this first: the upstream release does NOT do this. Every released config
# sets eval_interval: 9999 and eval_on_startup: false, so evaluation never fires
# during training, and no evaluation entry point is shipped. The paper's
# "Completion Rate" panel comes from the TRAINING rollout metric
# rollout/env_done/mean (analysis/plot_training_dynamics.py in the release).
# This script is an addition, not a reproduction of their protocol.
#
# What it does: renders configs/bench.yaml.tmpl for the chosen run and launches
# trinity's `bench` mode, which loads each outputs/checkpoints/.../global_step_*
# in turn and runs the full eval set (temperature 0.4, sequential over all 1,308
# tasks, max 30 turns).
#
# Two properties worth knowing:
# * It benches every method through the SAME workflow (the OPD one). That is
# deliberate: run through their own workflows, Guided-OPD would apply its
# 10-turn sliding context window while the other four accumulate, so the
# arms would not be comparable. One shared bench workflow makes them so.
# * Metrics are env_rounds / env_done / kl_divergence. There is no score
# metric: ScienceWorld's 0-100 partial credit is computed and stored on
# Experience.reward, which never reaches the metrics dict, in this bundle
# or upstream. env_done is the task-performance signal, and it counts any
# terminating episode -- including failures, which ScienceWorld terminates
# by driving the score negative.
set -euo pipefail
METHOD=${1:-}
BUNDLE=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
WITH_BASE=false
for a in "$@"; do [ "$a" = "--with-base" ] && WITH_BASE=true; done
case "$METHOD" in
ftb) RUN_NAME=scienceworld_ftb_qwen3_32b_to_1_7b ;;
opd) RUN_NAME=scienceworld_opd_qwen3_32b_to_1_7b ;;
guided_opd) RUN_NAME=scienceworld_guided_opd_qwen3_32b_to_1_7b ;;
tcod_b2f) RUN_NAME=scienceworld_tcod_b2f_qwen3_32b_to_1_7b ;;
tcod_f2b) RUN_NAME=scienceworld_tcod_f2b_qwen3_32b_to_1_7b ;;
*) echo "usage: $0 <ftb|opd|guided_opd|tcod_b2f|tcod_f2b> [--with-base]" >&2; exit 2 ;;
esac
cd "$BUNDLE"
CKPT_DIR="outputs/checkpoints/FutureBridge-OPD/$RUN_NAME"
if ! ls -d "$CKPT_DIR"/global_step_* >/dev/null 2>&1 && [ "$WITH_BASE" != true ]; then
echo "[eval] no checkpoints under $CKPT_DIR -- train first, or pass" >&2
echo "[eval] --with-base to score only the untrained base model" >&2
exit 1
fi
mkdir -p outputs/logs
CFG="outputs/bench_${RUN_NAME}.yaml"
RUN_NAME="$RUN_NAME" EVAL_ON_STARTUP="$WITH_BASE" \
python - configs/bench.yaml.tmpl "$CFG" <<'EOF'
import os, string, sys
tmpl = string.Template(open(sys.argv[1]).read())
open(sys.argv[2], "w").write(tmpl.substitute(
RUN_NAME=os.environ["RUN_NAME"],
EVAL_ON_STARTUP=os.environ["EVAL_ON_STARTUP"],
))
EOF
echo "[eval] rendered $CFG (run=$RUN_NAME, with_base=$WITH_BASE)"
if ! ray status >/dev/null 2>&1; then
echo "[eval] starting a local ray head"
ray start --head
fi
LOG="outputs/logs/bench_${METHOD}_$(date +%Y%m%d_%H%M%S).log"
echo "[eval] log=$LOG"
trinity run --config "$CFG" 2>&1 | tee "$LOG"
echo
echo "[eval] context-truncation summary for this bench run:"
echo "[eval] $(grep -c 'Prompt was truncated to' "$LOG" || true) turns exceeded max_prompt_tokens."
echo "[eval] Those turns generate nothing (the engine is skipped), so the agent"
echo "[eval] stops acting for the rest of that episode. Report this alongside"
echo "[eval] env_done -- it caps the effective turn budget."
|