File size: 3,908 Bytes
76b7b25 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 | #!/usr/bin/env python3
from __future__ import annotations
import argparse
import csv
import json
from pathlib import Path
from common import cfg_path, load_config, select_models
QUALITY_COLUMNS = [
"visual_quality",
"motion_quality",
"audio_quality",
"ib_tv",
"ib_ta",
"ib_av",
"clip_score",
"clap_score",
"cavp_score",
"av_align",
"avh_score",
"javis_score",
"desync",
]
DIVERSITY_COLUMNS = [
"video_pairwise_cosine_distance",
"audio_pairwise_cosine_distance",
"va_average_pairwise_cosine_distance",
"joint_pairwise_cosine_distance",
"video_pairwise_l2_distance",
"audio_pairwise_l2_distance",
"joint_pairwise_l2_distance",
]
def read_json(path: Path) -> dict:
if not path.is_file():
return {}
with path.open(encoding="utf-8") as f:
payload = json.load(f)
if isinstance(payload, dict):
return payload
return {}
def read_diversity_summary(path: Path) -> dict[str, dict]:
if not path.is_file():
return {}
with path.open(newline="", encoding="utf-8") as f:
rows = {}
for row in csv.DictReader(f):
name = row.get("model_name") or row.get("row_name")
if name:
rows[name] = row
return rows
def fmt(value) -> str:
if value is None or value == "":
return ""
try:
return f"{float(value):.4f}"
except (TypeError, ValueError):
return str(value)
def write_csv(path: Path, rows: list[dict], columns: list[str]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=columns)
writer.writeheader()
writer.writerows(rows)
def write_markdown(path: Path, rows: list[dict], columns: list[str]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
lines = []
lines.append("| " + " | ".join(columns) + " |")
lines.append("| " + " | ".join(["---"] * len(columns)) + " |")
for row in rows:
lines.append("| " + " | ".join(str(row.get(col, "")) for col in columns) + " |")
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--config", required=True)
parser.add_argument("--only", default=None)
parser.add_argument("--roles", default=None)
parser.add_argument("--output-prefix", default=None)
return parser.parse_args()
def main() -> int:
args = parse_args()
config = load_config(args.config)
models = select_models(config, args.only, args.roles)
quality_root = cfg_path(config, "quality_output_root")
diversity_summary = read_diversity_summary(cfg_path(config, "diversity_metric_root") / "summary.csv")
rows = []
for model in models:
name = model["name"]
quality = read_json(quality_root / f"{name}.json")
diversity = diversity_summary.get(name, {})
row = {
"model": name,
"display_name": model.get("display_name", ""),
"role": model.get("role", ""),
}
for col in QUALITY_COLUMNS:
row[col] = fmt(quality.get(col))
for col in DIVERSITY_COLUMNS:
row[col] = fmt(diversity.get(col))
rows.append(row)
columns = ["model", "display_name", "role", *QUALITY_COLUMNS, *DIVERSITY_COLUMNS]
local_root = cfg_path(config, "local_results_root", required=False) or Path("results")
prefix = Path(args.output_prefix) if args.output_prefix else local_root / "metrics_summary"
write_csv(prefix.with_suffix(".csv"), rows, columns)
write_markdown(prefix.with_suffix(".md"), rows, columns)
print(prefix.with_suffix(".csv"))
print(prefix.with_suffix(".md"))
return 0
if __name__ == "__main__":
raise SystemExit(main())
|