turbot2va-ltx2 / code /evaluation /scripts /plot_ablation_curve.py
luyu1021's picture
Add documented LTX-2 training source, recipes and checkpoint release plan
76b7b25 verified
Raw History Blame Contribute Delete
5.08 kB
#!/usr/bin/env python3
"""Aggregate scored checkpoint points and plot local Javis/AVH training curves."""
from __future__ import annotations
import argparse
import csv
import json
import math
import re
from pathlib import Path
def collect(root: Path) -> list[dict]:
points: list[dict] = []
for manifest_path in sorted(root.glob("*/checkpoint_*/evaluation_manifest.json")):
output = manifest_path.parent
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
stage = Path(manifest["run"]).name
match = re.fullmatch(r"checkpoint_(\d+)", output.name)
if not match or output.parent.name != stage:
raise ValueError(f"Unexpected output path: {output}")
step = int(match[1])
if manifest["selected_steps"] != [step]:
raise ValueError(f"Checkpoint/manifest mismatch: {manifest_path}")
score_path = output / "scores" / f"checkpoint_{step:06d}.json"
if not score_path.is_file():
continue
score = json.loads(score_path.read_text(encoding="utf-8"))
if any(
not isinstance(score.get(key), (int, float))
or isinstance(score[key], bool) or not math.isfinite(score[key])
for key in ("javis_score", "avh_score")
):
continue
offset = int(manifest["step_offset"])
batch = int(manifest["batch_size"])
if stage.endswith("_rcm") and offset != 1250:
raise ValueError(f"Expected 1250 warm-up local steps before rCM: {stage}")
if not stage.endswith("_rcm") and offset != 0:
raise ValueError(f"Unexpected warm-up offset: {stage}")
points.append({
"variant": stage.split("_")[0],
"stage": stage,
"local_step": step,
"stage_offset_local": offset,
"equivalent_paper_8gpu_steps": (step + offset) * batch // 8,
"samples_seen": (step + offset) * batch,
"num_prompts": manifest["num_prompts"],
"prompt_sha256": manifest["prompt_sha256"],
"seed": manifest["seed"],
"javis_score": score["javis_score"],
"avh_score": score["avh_score"],
"score_file": str(score_path),
})
points.sort(key=lambda row: (row["variant"], row["samples_seen"], row["stage"]))
if points and len({(p["num_prompts"], p["prompt_sha256"], p["seed"])
for p in points}) != 1:
raise ValueError("Cannot overlay curves from different prompt sets/seeds")
return points
def plot(root: Path, png_only: bool = False) -> None:
points = collect(root)
if not points:
print(f"No scored checkpoint points yet: {root}", flush=True)
return
destination = root / "combined_curve.csv"
with destination.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=list(points[0]))
writer.writeheader()
writer.writerows(points)
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
raw_dcm_200 = root.name == "exp1_dcm_8gpu_curve_raw200" and {
p["stage"] for p in points
} == {"exp1_dcm_8gpu"} and {p["num_prompts"] for p in points} == {200}
fig, axes = plt.subplots(1, 2, figsize=(10.5, 3.5))
for variant in sorted({p["variant"] for p in points}):
data = [p for p in points if p["variant"] == variant]
for axis, name in zip(axes, ("javis_score", "avh_score")):
axis.plot([p["equivalent_paper_8gpu_steps"] for p in data],
[p[name] for p in data], marker="o", label=variant)
for axis, name in zip(axes, ("Javis score", "AVH score")):
if not raw_dcm_200:
axis.axvline(2500, ls=":", color="gray", lw=1, label="warm-up ends")
axis.set(xlabel="Equivalent paper steps (8 GPUs × batch 1)", ylabel=name)
axis.grid(alpha=0.25)
axes[0].legend(fontsize=8)
if raw_dcm_200:
fig.suptitle(
"8-GPU dCM-only | 200 original (un-enhanced) prompts, "
"seed 12345, 4 inference steps\n"
"Paper Figure 5(a) switches to sCM at 2000; local 2500 remains dCM"
)
else:
fig.suptitle(
f"Local ablations | {points[0]['num_prompts']} enhanced prompts, "
"seed 12345, 4 inference steps\n"
"Different training runs from paper Figure 7; no original paper curve data"
)
fig.tight_layout()
for extension in (("png",) if png_only else ("png", "pdf")):
fig.savefig(root / f"combined_curve.{extension}", dpi=180)
plt.close(fig)
print(f"Saved {len(points)} points to {destination}", flush=True)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--root", type=Path, required=True)
parser.add_argument("--png-only", action="store_true",
help="Update the curve CSV and PNG without generating a PDF")
args = parser.parse_args()
plot(args.root.resolve(), png_only=args.png_only)