File size: 5,082 Bytes
76b7b25
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
#!/usr/bin/env python3
"""Aggregate scored checkpoint points and plot local Javis/AVH training curves."""
from __future__ import annotations

import argparse
import csv
import json
import math
import re
from pathlib import Path


def collect(root: Path) -> list[dict]:
    points: list[dict] = []
    for manifest_path in sorted(root.glob("*/checkpoint_*/evaluation_manifest.json")):
        output = manifest_path.parent
        manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
        stage = Path(manifest["run"]).name
        match = re.fullmatch(r"checkpoint_(\d+)", output.name)
        if not match or output.parent.name != stage:
            raise ValueError(f"Unexpected output path: {output}")
        step = int(match[1])
        if manifest["selected_steps"] != [step]:
            raise ValueError(f"Checkpoint/manifest mismatch: {manifest_path}")
        score_path = output / "scores" / f"checkpoint_{step:06d}.json"
        if not score_path.is_file():
            continue
        score = json.loads(score_path.read_text(encoding="utf-8"))
        if any(
            not isinstance(score.get(key), (int, float))
            or isinstance(score[key], bool) or not math.isfinite(score[key])
            for key in ("javis_score", "avh_score")
        ):
            continue
        offset = int(manifest["step_offset"])
        batch = int(manifest["batch_size"])
        if stage.endswith("_rcm") and offset != 1250:
            raise ValueError(f"Expected 1250 warm-up local steps before rCM: {stage}")
        if not stage.endswith("_rcm") and offset != 0:
            raise ValueError(f"Unexpected warm-up offset: {stage}")
        points.append({
            "variant": stage.split("_")[0],
            "stage": stage,
            "local_step": step,
            "stage_offset_local": offset,
            "equivalent_paper_8gpu_steps": (step + offset) * batch // 8,
            "samples_seen": (step + offset) * batch,
            "num_prompts": manifest["num_prompts"],
            "prompt_sha256": manifest["prompt_sha256"],
            "seed": manifest["seed"],
            "javis_score": score["javis_score"],
            "avh_score": score["avh_score"],
            "score_file": str(score_path),
        })
    points.sort(key=lambda row: (row["variant"], row["samples_seen"], row["stage"]))
    if points and len({(p["num_prompts"], p["prompt_sha256"], p["seed"])
                       for p in points}) != 1:
        raise ValueError("Cannot overlay curves from different prompt sets/seeds")
    return points


def plot(root: Path, png_only: bool = False) -> None:
    points = collect(root)
    if not points:
        print(f"No scored checkpoint points yet: {root}", flush=True)
        return
    destination = root / "combined_curve.csv"
    with destination.open("w", encoding="utf-8", newline="") as handle:
        writer = csv.DictWriter(handle, fieldnames=list(points[0]))
        writer.writeheader()
        writer.writerows(points)

    import matplotlib
    matplotlib.use("Agg")
    import matplotlib.pyplot as plt

    raw_dcm_200 = root.name == "exp1_dcm_8gpu_curve_raw200" and {
        p["stage"] for p in points
    } == {"exp1_dcm_8gpu"} and {p["num_prompts"] for p in points} == {200}
    fig, axes = plt.subplots(1, 2, figsize=(10.5, 3.5))
    for variant in sorted({p["variant"] for p in points}):
        data = [p for p in points if p["variant"] == variant]
        for axis, name in zip(axes, ("javis_score", "avh_score")):
            axis.plot([p["equivalent_paper_8gpu_steps"] for p in data],
                      [p[name] for p in data], marker="o", label=variant)
    for axis, name in zip(axes, ("Javis score", "AVH score")):
        if not raw_dcm_200:
            axis.axvline(2500, ls=":", color="gray", lw=1, label="warm-up ends")
        axis.set(xlabel="Equivalent paper steps (8 GPUs × batch 1)", ylabel=name)
        axis.grid(alpha=0.25)
    axes[0].legend(fontsize=8)
    if raw_dcm_200:
        fig.suptitle(
            "8-GPU dCM-only | 200 original (un-enhanced) prompts, "
            "seed 12345, 4 inference steps\n"
            "Paper Figure 5(a) switches to sCM at 2000; local 2500 remains dCM"
        )
    else:
        fig.suptitle(
            f"Local ablations | {points[0]['num_prompts']} enhanced prompts, "
            "seed 12345, 4 inference steps\n"
            "Different training runs from paper Figure 7; no original paper curve data"
        )
    fig.tight_layout()
    for extension in (("png",) if png_only else ("png", "pdf")):
        fig.savefig(root / f"combined_curve.{extension}", dpi=180)
    plt.close(fig)
    print(f"Saved {len(points)} points to {destination}", flush=True)


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--root", type=Path, required=True)
    parser.add_argument("--png-only", action="store_true",
                        help="Update the curve CSV and PNG without generating a PDF")
    args = parser.parse_args()
    plot(args.root.resolve(), png_only=args.png_only)