scripts/eval.sh

#!/usr/bin/env bash
# OPTIONAL. Score a finished run's checkpoints on the 1,308-task test split.
#
#     bash scripts/eval.sh <method> [--with-base]
#
# Read this first: the upstream release does NOT do this. Every released config
# sets eval_interval: 9999 and eval_on_startup: false, so evaluation never fires
# during training, and no evaluation entry point is shipped. The paper's
# "Completion Rate" panel comes from the TRAINING rollout metric
# rollout/env_done/mean (analysis/plot_training_dynamics.py in the release).
# This script is an addition, not a reproduction of their protocol.
#
# What it does: renders configs/bench.yaml.tmpl for the chosen run and launches
# trinity's `bench` mode, which loads each outputs/checkpoints/.../global_step_*
# in turn and runs the full eval set (temperature 0.4, sequential over all 1,308
# tasks, max 30 turns).
#
# Two properties worth knowing:
#   * It benches every method through the SAME workflow (the OPD one). That is
#     deliberate: run through their own workflows, Guided-OPD would apply its
#     10-turn sliding context window while the other four accumulate, so the
#     arms would not be comparable. One shared bench workflow makes them so.
#   * Metrics are env_rounds / env_done / kl_divergence. There is no score
#     metric: ScienceWorld's 0-100 partial credit is computed and stored on
#     Experience.reward, which never reaches the metrics dict, in this bundle
#     or upstream. env_done is the task-performance signal, and it counts any
#     terminating episode -- including failures, which ScienceWorld terminates
#     by driving the score negative.

set -euo pipefail

METHOD=${1:-}
BUNDLE=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
WITH_BASE=false
for a in "$@"; do [ "$a" = "--with-base" ] && WITH_BASE=true; done

case "$METHOD" in
    ftb)        RUN_NAME=scienceworld_ftb_qwen3_32b_to_1_7b ;;
    opd)        RUN_NAME=scienceworld_opd_qwen3_32b_to_1_7b ;;
    guided_opd) RUN_NAME=scienceworld_guided_opd_qwen3_32b_to_1_7b ;;
    tcod_b2f)   RUN_NAME=scienceworld_tcod_b2f_qwen3_32b_to_1_7b ;;
    tcod_f2b)   RUN_NAME=scienceworld_tcod_f2b_qwen3_32b_to_1_7b ;;
    *) echo "usage: $0 <ftb|opd|guided_opd|tcod_b2f|tcod_f2b> [--with-base]" >&2; exit 2 ;;
esac

cd "$BUNDLE"
CKPT_DIR="outputs/checkpoints/FutureBridge-OPD/$RUN_NAME"
if ! ls -d "$CKPT_DIR"/global_step_* >/dev/null 2>&1 && [ "$WITH_BASE" != true ]; then
    echo "[eval] no checkpoints under $CKPT_DIR -- train first, or pass" >&2
    echo "[eval] --with-base to score only the untrained base model" >&2
    exit 1
fi

mkdir -p outputs/logs
CFG="outputs/bench_${RUN_NAME}.yaml"
RUN_NAME="$RUN_NAME" EVAL_ON_STARTUP="$WITH_BASE" \
    python - configs/bench.yaml.tmpl "$CFG" <<'EOF'
import os, string, sys
tmpl = string.Template(open(sys.argv[1]).read())
open(sys.argv[2], "w").write(tmpl.substitute(
    RUN_NAME=os.environ["RUN_NAME"],
    EVAL_ON_STARTUP=os.environ["EVAL_ON_STARTUP"],
))
EOF
echo "[eval] rendered $CFG (run=$RUN_NAME, with_base=$WITH_BASE)"

if ! ray status >/dev/null 2>&1; then
    echo "[eval] starting a local ray head"
    ray start --head
fi

LOG="outputs/logs/bench_${METHOD}_$(date +%Y%m%d_%H%M%S).log"
echo "[eval] log=$LOG"
trinity run --config "$CFG" 2>&1 | tee "$LOG"

echo
echo "[eval] context-truncation summary for this bench run:"
echo "[eval]   $(grep -c 'Prompt was truncated to' "$LOG" || true) turns exceeded max_prompt_tokens."
echo "[eval]   Those turns generate nothing (the engine is skipped), so the agent"
echo "[eval]   stops acting for the rest of that episode. Report this alongside"
echo "[eval]   env_done -- it caps the effective turn budget."