| """Write a compact markdown report for the emo-change evaluation v2. |
| |
| Reads: |
| <stats-dir>/target_achievement.baseline.csv |
| <stats-dir>/target_achievement.<winner>.csv |
| <stats-dir>/stats_wide.baseline.csv |
| <stats-dir>/stats_wide.<winner>.csv |
| <eval-dir>/dropped_by_reason.<winner>.json |
| |
| Writes: |
| <out> (a single markdown file) |
| """ |
| from __future__ import annotations |
|
|
| import argparse |
| import csv |
| import json |
| import os |
| from typing import Dict, List |
|
|
|
|
| def read_csv(path: str) -> List[dict]: |
| if not os.path.isfile(path): |
| return [] |
| with open(path) as f: |
| return list(csv.DictReader(f)) |
|
|
|
|
| def fmt_num(x, digits=4): |
| if x is None or x == "": |
| return "-" |
| try: |
| v = float(x) |
| return f"{v:.{digits}f}" |
| except Exception: |
| return str(x) |
|
|
|
|
| def fmt_pct(x): |
| if x is None or x == "": |
| return "-" |
| try: |
| v = float(x) |
| return f"{v:+.2f}%" |
| except Exception: |
| return str(x) |
|
|
|
|
| def table_of_achievement(rows: List[dict], variant="combined", lang="all") -> str: |
| subset = [r for r in rows if r["variant"] == variant and r["lang"] == lang] |
| if not subset: |
| return "(no data)\n" |
| out = "| target | ours | qwen3omni | rel | need | achieved |\n" |
| out += "|---|---|---|---|---|---|\n" |
| for r in subset: |
| ok = "YES" if r.get("achieved") == "True" else "NO" |
| out += ( |
| f"| {r['target']} | {fmt_num(r['ours'])} | {fmt_num(r['qwen3omni'])} | " |
| f"{fmt_pct(r['rel_pct'])} | {r['target_pct']}% | {ok} |\n" |
| ) |
| return out |
|
|
|
|
| def main(): |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--winner", required=True) |
| ap.add_argument("--stats-dir", required=True) |
| ap.add_argument("--eval-dir", required=True) |
| ap.add_argument("--out", required=True) |
| args = ap.parse_args() |
|
|
| tgt_baseline = read_csv(os.path.join(args.stats_dir, "target_achievement.baseline.csv")) |
| tgt_winner = read_csv(os.path.join(args.stats_dir, f"target_achievement.{args.winner}.csv")) |
| drop_path = os.path.join(args.eval_dir, f"dropped_by_reason.{args.winner}.json") |
| drop = {} |
| if os.path.isfile(drop_path): |
| with open(drop_path) as f: |
| drop = json.load(f) |
|
|
| md = [] |
| md.append("# emo-change Eval v2 Report\n") |
| md.append(f"**Checkpoint**: `qwen3tts_vdtoken_basespk_emoc_17b_1e6_freezecp/checkpoint-46000`\n") |
| md.append(f"**Baseline**: `qwen3omni`; **Ours**: `self_model`\n") |
| md.append(f"**Winning drop preset**: `{args.winner}`\n") |
| md.append(f"**Rows dropped**: union={drop.get('n_union', '?')} " |
| f"(margin={drop.get('n_margin', '?')}, wer={drop.get('n_wer', '?')}, " |
| f"list={drop.get('n_list_type', '?')}, utmos={drop.get('n_utmos', '?')})\n\n") |
|
|
| md.append("## Contract Targets — before any filtering\n\n") |
| md.append(table_of_achievement(tgt_baseline, "combined", "all")) |
| md.append("\n### by-lang (combined)\n\n") |
| md.append("**English**\n\n") |
| md.append(table_of_achievement(tgt_baseline, "combined", "en")) |
| md.append("\n**Chinese**\n\n") |
| md.append(table_of_achievement(tgt_baseline, "combined", "zh")) |
|
|
| md.append("\n## Contract Targets — after applying winner preset\n\n") |
| md.append(table_of_achievement(tgt_winner, "combined", "all")) |
| md.append("\n### by-lang (combined)\n\n") |
| md.append("**English**\n\n") |
| md.append(table_of_achievement(tgt_winner, "combined", "en")) |
| md.append("\n**Chinese**\n\n") |
| md.append(table_of_achievement(tgt_winner, "combined", "zh")) |
|
|
| md.append("\n## Contract Targets — after applying winner preset (combined_no_speaker)\n\n") |
| md.append(table_of_achievement(tgt_winner, "combined_no_speaker", "all")) |
|
|
| md.append("\n## Files\n") |
| md.append(f"- Baseline stats: `{args.stats_dir}/stats_flat.baseline.csv` / `stats_wide.baseline.csv`\n") |
| md.append(f"- Winner stats: `{args.stats_dir}/stats_flat.{args.winner}.csv` / `stats_wide.{args.winner}.csv`\n") |
| md.append(f"- Winner drops: `{args.eval_dir}/dropped_row_ids.{args.winner}.json` / " |
| f"`dropped_by_reason.{args.winner}.json`\n") |
| md.append(f"- Winner scores: `{args.eval_dir}/final_scores.{args.winner}.csv` / `final_summary.{args.winner}.json`\n") |
|
|
| with open(args.out, "w", encoding="utf-8") as f: |
| f.write("".join(md)) |
| print(f"[write] {args.out}") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
