#!/usr/bin/env python3 from __future__ import annotations import argparse import csv import json from pathlib import Path from common import cfg_path, load_config, select_models QUALITY_COLUMNS = [ "visual_quality", "motion_quality", "audio_quality", "ib_tv", "ib_ta", "ib_av", "clip_score", "clap_score", "cavp_score", "av_align", "avh_score", "javis_score", "desync", ] DIVERSITY_COLUMNS = [ "video_pairwise_cosine_distance", "audio_pairwise_cosine_distance", "va_average_pairwise_cosine_distance", "joint_pairwise_cosine_distance", "video_pairwise_l2_distance", "audio_pairwise_l2_distance", "joint_pairwise_l2_distance", ] def read_json(path: Path) -> dict: if not path.is_file(): return {} with path.open(encoding="utf-8") as f: payload = json.load(f) if isinstance(payload, dict): return payload return {} def read_diversity_summary(path: Path) -> dict[str, dict]: if not path.is_file(): return {} with path.open(newline="", encoding="utf-8") as f: rows = {} for row in csv.DictReader(f): name = row.get("model_name") or row.get("row_name") if name: rows[name] = row return rows def fmt(value) -> str: if value is None or value == "": return "" try: return f"{float(value):.4f}" except (TypeError, ValueError): return str(value) def write_csv(path: Path, rows: list[dict], columns: list[str]) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", newline="", encoding="utf-8") as f: writer = csv.DictWriter(f, fieldnames=columns) writer.writeheader() writer.writerows(rows) def write_markdown(path: Path, rows: list[dict], columns: list[str]) -> None: path.parent.mkdir(parents=True, exist_ok=True) lines = [] lines.append("| " + " | ".join(columns) + " |") lines.append("| " + " | ".join(["---"] * len(columns)) + " |") for row in rows: lines.append("| " + " | ".join(str(row.get(col, "")) for col in columns) + " |") path.write_text("\n".join(lines) + "\n", encoding="utf-8") def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--config", required=True) parser.add_argument("--only", default=None) parser.add_argument("--roles", default=None) parser.add_argument("--output-prefix", default=None) return parser.parse_args() def main() -> int: args = parse_args() config = load_config(args.config) models = select_models(config, args.only, args.roles) quality_root = cfg_path(config, "quality_output_root") diversity_summary = read_diversity_summary(cfg_path(config, "diversity_metric_root") / "summary.csv") rows = [] for model in models: name = model["name"] quality = read_json(quality_root / f"{name}.json") diversity = diversity_summary.get(name, {}) row = { "model": name, "display_name": model.get("display_name", ""), "role": model.get("role", ""), } for col in QUALITY_COLUMNS: row[col] = fmt(quality.get(col)) for col in DIVERSITY_COLUMNS: row[col] = fmt(diversity.get(col)) rows.append(row) columns = ["model", "display_name", "role", *QUALITY_COLUMNS, *DIVERSITY_COLUMNS] local_root = cfg_path(config, "local_results_root", required=False) or Path("results") prefix = Path(args.output_prefix) if args.output_prefix else local_root / "metrics_summary" write_csv(prefix.with_suffix(".csv"), rows, columns) write_markdown(prefix.with_suffix(".md"), rows, columns) print(prefix.with_suffix(".csv")) print(prefix.with_suffix(".md")) return 0 if __name__ == "__main__": raise SystemExit(main())