turbot2va-ltx2 / code /evaluation /scripts /summarize_results.py
luyu1021's picture
Add documented LTX-2 training source, recipes and checkpoint release plan
76b7b25 verified
Raw History Blame Contribute Delete
3.91 kB
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import csv
import json
from pathlib import Path
from common import cfg_path, load_config, select_models
QUALITY_COLUMNS = [
"visual_quality",
"motion_quality",
"audio_quality",
"ib_tv",
"ib_ta",
"ib_av",
"clip_score",
"clap_score",
"cavp_score",
"av_align",
"avh_score",
"javis_score",
"desync",
]
DIVERSITY_COLUMNS = [
"video_pairwise_cosine_distance",
"audio_pairwise_cosine_distance",
"va_average_pairwise_cosine_distance",
"joint_pairwise_cosine_distance",
"video_pairwise_l2_distance",
"audio_pairwise_l2_distance",
"joint_pairwise_l2_distance",
]
def read_json(path: Path) -> dict:
if not path.is_file():
return {}
with path.open(encoding="utf-8") as f:
payload = json.load(f)
if isinstance(payload, dict):
return payload
return {}
def read_diversity_summary(path: Path) -> dict[str, dict]:
if not path.is_file():
return {}
with path.open(newline="", encoding="utf-8") as f:
rows = {}
for row in csv.DictReader(f):
name = row.get("model_name") or row.get("row_name")
if name:
rows[name] = row
return rows
def fmt(value) -> str:
if value is None or value == "":
return ""
try:
return f"{float(value):.4f}"
except (TypeError, ValueError):
return str(value)
def write_csv(path: Path, rows: list[dict], columns: list[str]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=columns)
writer.writeheader()
writer.writerows(rows)
def write_markdown(path: Path, rows: list[dict], columns: list[str]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
lines = []
lines.append("| " + " | ".join(columns) + " |")
lines.append("| " + " | ".join(["---"] * len(columns)) + " |")
for row in rows:
lines.append("| " + " | ".join(str(row.get(col, "")) for col in columns) + " |")
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--config", required=True)
parser.add_argument("--only", default=None)
parser.add_argument("--roles", default=None)
parser.add_argument("--output-prefix", default=None)
return parser.parse_args()
def main() -> int:
args = parse_args()
config = load_config(args.config)
models = select_models(config, args.only, args.roles)
quality_root = cfg_path(config, "quality_output_root")
diversity_summary = read_diversity_summary(cfg_path(config, "diversity_metric_root") / "summary.csv")
rows = []
for model in models:
name = model["name"]
quality = read_json(quality_root / f"{name}.json")
diversity = diversity_summary.get(name, {})
row = {
"model": name,
"display_name": model.get("display_name", ""),
"role": model.get("role", ""),
}
for col in QUALITY_COLUMNS:
row[col] = fmt(quality.get(col))
for col in DIVERSITY_COLUMNS:
row[col] = fmt(diversity.get(col))
rows.append(row)
columns = ["model", "display_name", "role", *QUALITY_COLUMNS, *DIVERSITY_COLUMNS]
local_root = cfg_path(config, "local_results_root", required=False) or Path("results")
prefix = Path(args.output_prefix) if args.output_prefix else local_root / "metrics_summary"
write_csv(prefix.with_suffix(".csv"), rows, columns)
write_markdown(prefix.with_suffix(".md"), rows, columns)
print(prefix.with_suffix(".csv"))
print(prefix.with_suffix(".md"))
return 0
if __name__ == "__main__":
raise SystemExit(main())