hy_aggregate.py

#!/usr/bin/env python
"""Summarise the HY-WorldPlay evaluation into eval_out_hy/summaries/final_summary.{json,csv}.

Latency: ratio of means against the FFFF latency measured in the same process for
the same job (matched FFFF).  Pixel metrics: PSNR from the mean per-video MSE, SSIM
and LPIPS averaged.  VBench: the eight validation25 dimensions, normalised with
VBench's official ranges; the headline "VBench-I2V-100" is the 251-protocol
formula with temporal_flickering in the quality group and overall_consistency as
the semantic group (there is no scene dimension for I2V):
    quality  = (sc + bc + tf + ms + 0.5*dd + aq + iq) / 6.5
    semantic = oc
    selected = (4*quality + semantic) / 5
"core5" (mean of raw sc, bc, ms, aq, iq) is kept for comparison with the DEV
project's tables.
"""
import csv, glob, json, math, os, sys
ROOT = os.path.dirname(os.path.abspath(__file__))
OUT = os.path.join(ROOT, "eval_out_hy")
# Two headline numbers over the same five dimensions, both reported:
#
#   core5     the plain mean of the raw scores -- the HY-WorldPlay DEV protocol's
#             "VBench Core5" for validation25, so the tables line up with it;
#   score_5d  the mean after VBench's *official* min-max normalisation, which is
#             what the 251-prompt tables use for their quality group.
#
# VBench reports imaging_quality *per video* on 0..100 but already divides its
# own aggregate by 100, and the aggregate is what is collected here -- so no
# further scaling is applied (dividing again would cost ~14 points of Core5).
# aesthetic_quality and imaging_quality live on [0, 1], so their official
# normalisation is the identity.
DIMS = ["subject_consistency", "background_consistency", "motion_smoothness",
        "aesthetic_quality", "imaging_quality"]
NORM_RANGE = {"subject_consistency": (0.1462, 1.0),
              "background_consistency": (0.2615, 1.0),
              "motion_smoothness": (0.7060, 0.9975),
              "aesthetic_quality": (0.0, 1.0),
              "imaging_quality": (0.0, 1.0)}


def normalize_score(dim, value):
    return value


def official_normalize(dim, value):
    lo, hi = NORM_RANGE[dim]
    return (value - lo) / (hi - lo)
only = sys.argv[1] if len(sys.argv) > 1 else None

rows, problems = [], []
for d in sorted(glob.glob(os.path.join(OUT, "per_prompt", "hy_*"))):
    s = os.path.basename(d)
    recs = [json.load(open(p)) for p in sorted(glob.glob(os.path.join(d, "*.json")))]
    recs = [r for r in recs if r.get("status") == "complete"]
    if len(recs) != 100:
        problems.append(f"{s}: {len(recs)} records"); continue
    sc = os.path.join(OUT, "vbench", "scores", s + ".json")
    vb = json.load(open(sc)) if os.path.exists(sc) else None
    if vb is None:
        problems.append(f"{s}: no VBench scores")
    lat = sum(r["policy_latency_ms"] for r in recs) / 100
    matched = [r.get("matched_ffff_policy_latency_ms") for r in recs]
    if all(m is not None for m in matched):
        ref = sum(matched) / 100
        latency_source = "paired with FFFF in the same process"
    else:
        # FFFF was not regenerated for these jobs: compare against the reference
        # strategy's own records (hy_ffff, or hy_<n>c_ffff for the long videos).
        ref_name = recs[0].get("reference_strategy") or "hy_ffff"
        ff = [json.load(open(p)) for p in sorted(glob.glob(os.path.join(OUT, "per_prompt", ref_name, "*.json")))]
        ref = sum(r["policy_latency_ms"] for r in ff if r.get("status") == "complete") / 100
        latency_source = "generation run (FFFF from its own records)"
        if all(str(r.get("latency_source", "")).startswith("DEV generator") for r in recs):
            latency_source = recs[0]["latency_source"]
    ce = sum(r["cache_diagnostics"]["compute_equivalent_forwards"] for r in recs) / 100
    fw = recs[0]["cache_diagnostics"]["denoise_forwards"]
    mse = sum(r["pixel_metrics_vs_ffff"]["mean_mse"] for r in recs) / 100
    row = {"strategy": s, "base_model": "hy_worldplay", "method": recs[0]["method"],
           "schedule": recs[0]["schedule"], "target_speedup": recs[0]["target_speedup"],
           "param": recs[0]["param"], "param_value": recs[0]["param_value"], "num_videos": 100,
           "policy_latency_ms": lat, "matched_ffff_policy_latency_ms": ref,
           "latency_ratio_vs_ffff": lat / ref, "speedup_percent_vs_ffff": 100 * (1 - lat / ref),
           "latency_source": latency_source,
           "first_chunk_schedule": recs[0].get("first_chunk_schedule"),
           "mp4_reference_records": sum(1 for r in recs if r.get("reference_source") == "ffff_mp4"),
           "excluded_context_kv_latency_ms": sum(r["excluded_context_kv_latency_ms"] for r in recs) / 100,
           "denoise_forwards": fw, "mean_compute_equivalent_forwards": ce,
           "mean_full_forwards": (sum(r["cache_diagnostics"]["full_forwards"] for r in recs) / 100
                                  if all("full_forwards" in r["cache_diagnostics"] for r in recs) else None),
           "mean_predictor_forwards": (sum(r["cache_diagnostics"]["predictor_forwards"] for r in recs) / 100
                                       if all("predictor_forwards" in r["cache_diagnostics"] for r in recs) else None),
           "psnr": -10 * math.log10(max(mse, 1e-12)),
           "ssim": sum(r["pixel_metrics_vs_ffff"]["ssim"] for r in recs) / 100,
           "lpips": sum(r["pixel_metrics_vs_ffff"]["lpips"] for r in recs) / 100,
           "peak_mem_gib": max(r.get("peak_mem_gib", 0) for r in recs)}
    if vb:
        scores, normed = {}, {}
        for k in DIMS:
            row["raw_" + k] = vb["raw"][k]
            scores[k] = normalize_score(k, vb["raw"][k])
            normed[k] = official_normalize(k, scores[k])
            row["score_" + k] = scores[k]
            row["normalized_" + k] = normed[k]
        core5 = sum(scores.values()) / len(DIMS)
        score_5d = sum(normed.values()) / len(DIMS)
        row.update(core5=core5, core5_percent=100 * core5,
                   score_5d=score_5d, score_5d_percent=100 * score_5d,
                   selected_vbench_score=score_5d, selected_vbench_percent=100 * score_5d)
    rows.append(row)
rows.sort(key=lambda r: (["none", "teacache", "flowcache", "taylorseer", "motioncache"].index(r["method"])
                         if r["method"] in ("none", "teacache", "flowcache", "taylorseer", "motioncache") else 9,
                         r["schedule"] != "FxxF", r["target_speedup"]))
os.makedirs(os.path.join(OUT, "summaries"), exist_ok=True)
json.dump({"protocol": "HY-WorldPlay VBench-I2V-100 (validation25 x 4 actions, Core5)",
           "num_strategies": len(rows), "rows": rows, "problems": problems, "dimensions": DIMS,
           "aggregation": {
               "core5": "mean of raw(sc, bc, ms, aq, iq/100)",
               "score_5d": "mean of ((sc-0.1462)/0.8538, (bc-0.2615)/0.7385, "
                           "(ms-0.7060)/0.2915, aq, iq/100)"},
           "normalize_range": NORM_RANGE},
          open(os.path.join(OUT, "summaries", "final_summary.json"), "w"), indent=2)
keys = sorted({k for r in rows for k in r}, key=lambda k: (k not in ("strategy", "method"), k))
with open(os.path.join(OUT, "summaries", "final_summary.csv"), "w", newline="") as f:
    w = csv.DictWriter(f, fieldnames=keys); w.writeheader(); [w.writerow(r) for r in rows]
print(f"{'strategy':26s} {'lat_ms':>8} {'ffff_ms':>8} {'x':>6} {'Full':>6} {'PSNR':>6} {'LPIPS':>6} {'raw%':>7} {'norm%':>7}")
for r in rows:
    print(f"{r['strategy']:26s} {r['policy_latency_ms']:8.0f} {r['matched_ffff_policy_latency_ms']:8.0f} "
          f"{1/r['latency_ratio_vs_ffff']:6.3f} {r['mean_compute_equivalent_forwards']:6.2f} {r['psnr']:6.2f} "
          f"{r['lpips']:6.3f} {r.get('core5_percent', float('nan')):7.2f} "
          f"{r.get('score_5d_percent', float('nan')):7.2f}")
if problems: print("problems:", problems)