#!/usr/bin/env python3 """Aggregate scored checkpoint points and plot local Javis/AVH training curves.""" from __future__ import annotations import argparse import csv import json import math import re from pathlib import Path def collect(root: Path) -> list[dict]: points: list[dict] = [] for manifest_path in sorted(root.glob("*/checkpoint_*/evaluation_manifest.json")): output = manifest_path.parent manifest = json.loads(manifest_path.read_text(encoding="utf-8")) stage = Path(manifest["run"]).name match = re.fullmatch(r"checkpoint_(\d+)", output.name) if not match or output.parent.name != stage: raise ValueError(f"Unexpected output path: {output}") step = int(match[1]) if manifest["selected_steps"] != [step]: raise ValueError(f"Checkpoint/manifest mismatch: {manifest_path}") score_path = output / "scores" / f"checkpoint_{step:06d}.json" if not score_path.is_file(): continue score = json.loads(score_path.read_text(encoding="utf-8")) if any( not isinstance(score.get(key), (int, float)) or isinstance(score[key], bool) or not math.isfinite(score[key]) for key in ("javis_score", "avh_score") ): continue offset = int(manifest["step_offset"]) batch = int(manifest["batch_size"]) if stage.endswith("_rcm") and offset != 1250: raise ValueError(f"Expected 1250 warm-up local steps before rCM: {stage}") if not stage.endswith("_rcm") and offset != 0: raise ValueError(f"Unexpected warm-up offset: {stage}") points.append({ "variant": stage.split("_")[0], "stage": stage, "local_step": step, "stage_offset_local": offset, "equivalent_paper_8gpu_steps": (step + offset) * batch // 8, "samples_seen": (step + offset) * batch, "num_prompts": manifest["num_prompts"], "prompt_sha256": manifest["prompt_sha256"], "seed": manifest["seed"], "javis_score": score["javis_score"], "avh_score": score["avh_score"], "score_file": str(score_path), }) points.sort(key=lambda row: (row["variant"], row["samples_seen"], row["stage"])) if points and len({(p["num_prompts"], p["prompt_sha256"], p["seed"]) for p in points}) != 1: raise ValueError("Cannot overlay curves from different prompt sets/seeds") return points def plot(root: Path, png_only: bool = False) -> None: points = collect(root) if not points: print(f"No scored checkpoint points yet: {root}", flush=True) return destination = root / "combined_curve.csv" with destination.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=list(points[0])) writer.writeheader() writer.writerows(points) import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt raw_dcm_200 = root.name == "exp1_dcm_8gpu_curve_raw200" and { p["stage"] for p in points } == {"exp1_dcm_8gpu"} and {p["num_prompts"] for p in points} == {200} fig, axes = plt.subplots(1, 2, figsize=(10.5, 3.5)) for variant in sorted({p["variant"] for p in points}): data = [p for p in points if p["variant"] == variant] for axis, name in zip(axes, ("javis_score", "avh_score")): axis.plot([p["equivalent_paper_8gpu_steps"] for p in data], [p[name] for p in data], marker="o", label=variant) for axis, name in zip(axes, ("Javis score", "AVH score")): if not raw_dcm_200: axis.axvline(2500, ls=":", color="gray", lw=1, label="warm-up ends") axis.set(xlabel="Equivalent paper steps (8 GPUs × batch 1)", ylabel=name) axis.grid(alpha=0.25) axes[0].legend(fontsize=8) if raw_dcm_200: fig.suptitle( "8-GPU dCM-only | 200 original (un-enhanced) prompts, " "seed 12345, 4 inference steps\n" "Paper Figure 5(a) switches to sCM at 2000; local 2500 remains dCM" ) else: fig.suptitle( f"Local ablations | {points[0]['num_prompts']} enhanced prompts, " "seed 12345, 4 inference steps\n" "Different training runs from paper Figure 7; no original paper curve data" ) fig.tight_layout() for extension in (("png",) if png_only else ("png", "pdf")): fig.savefig(root / f"combined_curve.{extension}", dpi=180) plt.close(fig) print(f"Saved {len(points)} points to {destination}", flush=True) if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--root", type=Path, required=True) parser.add_argument("--png-only", action="store_true", help="Update the curve CSV and PNG without generating a PDF") args = parser.parse_args() plot(args.root.resolve(), png_only=args.png_only)