#!/usr/bin/env bash
set -euo pipefail

cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt

STAMP="${STAMP:?set STAMP used by run_train8_overfit_sweep_4gpu.sh}"
TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-standard/tokenizer.json}"
CACHE_DIR="${CACHE_DIR:-/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_overfit}"
OUT_ROOT="${OUT_ROOT:-docs/lta_samples/metrics_20260517/train8_overfit_${STAMP}}"
export OUT_ROOT
mkdir -p "${OUT_ROOT}"

for run in \
  "train8_n1024_hard_ce_onehot_${STAMP}" \
  "train8_n1024_hard_ce_bridge_${STAMP}" \
  "train8_n1024_linear_soft_kl_onehot_${STAMP}" \
  "train8_n1024_linear_soft_kl_bridge_${STAMP}"
do
  echo "[eval-overfit] ${run}"
  python scripts/eval_train8_overfit_ckpts.py \
    --run_dir "runs/${run}" \
    --cache_dir "${CACHE_DIR}" \
    --tokenizer_path "${TOKENIZER_PATH}" \
    --out_dir "${OUT_ROOT}/${run}" \
    --max_len 1024 \
    --limit 8 \
    --t_values 0.125,0.25,0.5,0.75,1.0 \
    --seeds 123,456,789
done

python - <<'PY'
import json
import os
from pathlib import Path
root = Path(os.environ["OUT_ROOT"])
rows = []
for p in sorted(root.glob("*/result.json")):
    r = json.loads(p.read_text())
    last = r["last"]
    rows.append({
        "run": p.parent.name,
        "first_fit_step": r["first_fit_step"],
        "best_acc": r["best_acc"],
        "best_objective": r["best_objective"],
        "last_acc": last["gold_acc_mean"],
        "last_objective": last["objective_mean"],
        "last_ce": last["gold_ce_mean"],
    })
with (root / "combined.tsv").open("w") as f:
    f.write("run\tfirst_fit_step\tbest_acc\tbest_objective\tlast_acc\tlast_objective\tlast_ce\n")
    for r in rows:
        f.write("\t".join(str(r[k]) for k in ["run","first_fit_step","best_acc","best_objective","last_acc","last_objective","last_ce"]) + "\n")
print(root / "combined.tsv")
PY