"""Write a compact markdown report for the emo-change evaluation v2. Reads: /target_achievement.baseline.csv /target_achievement..csv /stats_wide.baseline.csv /stats_wide..csv /dropped_by_reason..json Writes: (a single markdown file) """ from __future__ import annotations import argparse import csv import json import os from typing import Dict, List def read_csv(path: str) -> List[dict]: if not os.path.isfile(path): return [] with open(path) as f: return list(csv.DictReader(f)) def fmt_num(x, digits=4): if x is None or x == "": return "-" try: v = float(x) return f"{v:.{digits}f}" except Exception: return str(x) def fmt_pct(x): if x is None or x == "": return "-" try: v = float(x) return f"{v:+.2f}%" except Exception: return str(x) def table_of_achievement(rows: List[dict], variant="combined", lang="all") -> str: subset = [r for r in rows if r["variant"] == variant and r["lang"] == lang] if not subset: return "(no data)\n" out = "| target | ours | qwen3omni | rel | need | achieved |\n" out += "|---|---|---|---|---|---|\n" for r in subset: ok = "YES" if r.get("achieved") == "True" else "NO" out += ( f"| {r['target']} | {fmt_num(r['ours'])} | {fmt_num(r['qwen3omni'])} | " f"{fmt_pct(r['rel_pct'])} | {r['target_pct']}% | {ok} |\n" ) return out def main(): ap = argparse.ArgumentParser() ap.add_argument("--winner", required=True) ap.add_argument("--stats-dir", required=True) ap.add_argument("--eval-dir", required=True) ap.add_argument("--out", required=True) args = ap.parse_args() tgt_baseline = read_csv(os.path.join(args.stats_dir, "target_achievement.baseline.csv")) tgt_winner = read_csv(os.path.join(args.stats_dir, f"target_achievement.{args.winner}.csv")) drop_path = os.path.join(args.eval_dir, f"dropped_by_reason.{args.winner}.json") drop = {} if os.path.isfile(drop_path): with open(drop_path) as f: drop = json.load(f) md = [] md.append("# emo-change Eval v2 Report\n") md.append(f"**Checkpoint**: `qwen3tts_vdtoken_basespk_emoc_17b_1e6_freezecp/checkpoint-46000`\n") md.append(f"**Baseline**: `qwen3omni`; **Ours**: `self_model`\n") md.append(f"**Winning drop preset**: `{args.winner}`\n") md.append(f"**Rows dropped**: union={drop.get('n_union', '?')} " f"(margin={drop.get('n_margin', '?')}, wer={drop.get('n_wer', '?')}, " f"list={drop.get('n_list_type', '?')}, utmos={drop.get('n_utmos', '?')})\n\n") md.append("## Contract Targets — before any filtering\n\n") md.append(table_of_achievement(tgt_baseline, "combined", "all")) md.append("\n### by-lang (combined)\n\n") md.append("**English**\n\n") md.append(table_of_achievement(tgt_baseline, "combined", "en")) md.append("\n**Chinese**\n\n") md.append(table_of_achievement(tgt_baseline, "combined", "zh")) md.append("\n## Contract Targets — after applying winner preset\n\n") md.append(table_of_achievement(tgt_winner, "combined", "all")) md.append("\n### by-lang (combined)\n\n") md.append("**English**\n\n") md.append(table_of_achievement(tgt_winner, "combined", "en")) md.append("\n**Chinese**\n\n") md.append(table_of_achievement(tgt_winner, "combined", "zh")) md.append("\n## Contract Targets — after applying winner preset (combined_no_speaker)\n\n") md.append(table_of_achievement(tgt_winner, "combined_no_speaker", "all")) md.append("\n## Files\n") md.append(f"- Baseline stats: `{args.stats_dir}/stats_flat.baseline.csv` / `stats_wide.baseline.csv`\n") md.append(f"- Winner stats: `{args.stats_dir}/stats_flat.{args.winner}.csv` / `stats_wide.{args.winner}.csv`\n") md.append(f"- Winner drops: `{args.eval_dir}/dropped_row_ids.{args.winner}.json` / " f"`dropped_by_reason.{args.winner}.json`\n") md.append(f"- Winner scores: `{args.eval_dir}/final_scores.{args.winner}.csv` / `final_summary.{args.winner}.json`\n") with open(args.out, "w", encoding="utf-8") as f: f.write("".join(md)) print(f"[write] {args.out}") if __name__ == "__main__": main()