Download code/evaluation/scripts/plot_ablation_curve.py from luyu1021/turbot2va-ltx2: direct link, hf CLI and curl.
- Browser
- Download file 5.08 kB
-
https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/evaluation/scripts/plot_ablation_curve.py
- Command line
-
hf download hf://luyu1021/turbot2va-ltx2/code/evaluation/scripts/plot_ablation_curve.py
-
curl -L -o plot_ablation_curve.py https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/evaluation/scripts/plot_ablation_curve.py
5.08 kB
| #!/usr/bin/env python3 | |
| """Aggregate scored checkpoint points and plot local Javis/AVH training curves.""" | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import json | |
| import math | |
| import re | |
| from pathlib import Path | |
| def collect(root: Path) -> list[dict]: | |
| points: list[dict] = [] | |
| for manifest_path in sorted(root.glob("*/checkpoint_*/evaluation_manifest.json")): | |
| output = manifest_path.parent | |
| manifest = json.loads(manifest_path.read_text(encoding="utf-8")) | |
| stage = Path(manifest["run"]).name | |
| match = re.fullmatch(r"checkpoint_(\d+)", output.name) | |
| if not match or output.parent.name != stage: | |
| raise ValueError(f"Unexpected output path: {output}") | |
| step = int(match[1]) | |
| if manifest["selected_steps"] != [step]: | |
| raise ValueError(f"Checkpoint/manifest mismatch: {manifest_path}") | |
| score_path = output / "scores" / f"checkpoint_{step:06d}.json" | |
| if not score_path.is_file(): | |
| continue | |
| score = json.loads(score_path.read_text(encoding="utf-8")) | |
| if any( | |
| not isinstance(score.get(key), (int, float)) | |
| or isinstance(score[key], bool) or not math.isfinite(score[key]) | |
| for key in ("javis_score", "avh_score") | |
| ): | |
| continue | |
| offset = int(manifest["step_offset"]) | |
| batch = int(manifest["batch_size"]) | |
| if stage.endswith("_rcm") and offset != 1250: | |
| raise ValueError(f"Expected 1250 warm-up local steps before rCM: {stage}") | |
| if not stage.endswith("_rcm") and offset != 0: | |
| raise ValueError(f"Unexpected warm-up offset: {stage}") | |
| points.append({ | |
| "variant": stage.split("_")[0], | |
| "stage": stage, | |
| "local_step": step, | |
| "stage_offset_local": offset, | |
| "equivalent_paper_8gpu_steps": (step + offset) * batch // 8, | |
| "samples_seen": (step + offset) * batch, | |
| "num_prompts": manifest["num_prompts"], | |
| "prompt_sha256": manifest["prompt_sha256"], | |
| "seed": manifest["seed"], | |
| "javis_score": score["javis_score"], | |
| "avh_score": score["avh_score"], | |
| "score_file": str(score_path), | |
| }) | |
| points.sort(key=lambda row: (row["variant"], row["samples_seen"], row["stage"])) | |
| if points and len({(p["num_prompts"], p["prompt_sha256"], p["seed"]) | |
| for p in points}) != 1: | |
| raise ValueError("Cannot overlay curves from different prompt sets/seeds") | |
| return points | |
| def plot(root: Path, png_only: bool = False) -> None: | |
| points = collect(root) | |
| if not points: | |
| print(f"No scored checkpoint points yet: {root}", flush=True) | |
| return | |
| destination = root / "combined_curve.csv" | |
| with destination.open("w", encoding="utf-8", newline="") as handle: | |
| writer = csv.DictWriter(handle, fieldnames=list(points[0])) | |
| writer.writeheader() | |
| writer.writerows(points) | |
| import matplotlib | |
| matplotlib.use("Agg") | |
| import matplotlib.pyplot as plt | |
| raw_dcm_200 = root.name == "exp1_dcm_8gpu_curve_raw200" and { | |
| p["stage"] for p in points | |
| } == {"exp1_dcm_8gpu"} and {p["num_prompts"] for p in points} == {200} | |
| fig, axes = plt.subplots(1, 2, figsize=(10.5, 3.5)) | |
| for variant in sorted({p["variant"] for p in points}): | |
| data = [p for p in points if p["variant"] == variant] | |
| for axis, name in zip(axes, ("javis_score", "avh_score")): | |
| axis.plot([p["equivalent_paper_8gpu_steps"] for p in data], | |
| [p[name] for p in data], marker="o", label=variant) | |
| for axis, name in zip(axes, ("Javis score", "AVH score")): | |
| if not raw_dcm_200: | |
| axis.axvline(2500, ls=":", color="gray", lw=1, label="warm-up ends") | |
| axis.set(xlabel="Equivalent paper steps (8 GPUs × batch 1)", ylabel=name) | |
| axis.grid(alpha=0.25) | |
| axes[0].legend(fontsize=8) | |
| if raw_dcm_200: | |
| fig.suptitle( | |
| "8-GPU dCM-only | 200 original (un-enhanced) prompts, " | |
| "seed 12345, 4 inference steps\n" | |
| "Paper Figure 5(a) switches to sCM at 2000; local 2500 remains dCM" | |
| ) | |
| else: | |
| fig.suptitle( | |
| f"Local ablations | {points[0]['num_prompts']} enhanced prompts, " | |
| "seed 12345, 4 inference steps\n" | |
| "Different training runs from paper Figure 7; no original paper curve data" | |
| ) | |
| fig.tight_layout() | |
| for extension in (("png",) if png_only else ("png", "pdf")): | |
| fig.savefig(root / f"combined_curve.{extension}", dpi=180) | |
| plt.close(fig) | |
| print(f"Saved {len(points)} points to {destination}", flush=True) | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--root", type=Path, required=True) | |
| parser.add_argument("--png-only", action="store_true", | |
| help="Update the curve CSV and PNG without generating a PDF") | |
| args = parser.parse_args() | |
| plot(args.root.resolve(), png_only=args.png_only) | |