| |
| |
| """ |
| Generate a compact VStyle-style LaTeX table from metadata_with_score.jsonl files. |
| |
| Default layout: |
| scoring/outputs/<judge_tag>/<model>/<lang>/metadata_with_score.jsonl |
| """ |
|
|
| import argparse |
| import json |
| import math |
| import os |
| from collections import defaultdict |
| from pathlib import Path |
|
|
|
|
| SUB_CATS = { |
| "acoustic_attributes": [ |
| "acoustic_attributes/age", |
| "acoustic_attributes/speed", |
| "acoustic_attributes/gender", |
| "acoustic_attributes/emotion", |
| "acoustic_attributes/pitch", |
| "acoustic_attributes/volume", |
| "acoustic_attributes/composite_properties", |
| ], |
| "instruction": [ |
| "instruction/emotion", |
| "instruction/variation", |
| "instruction/style", |
| ], |
| "role_play": [ |
| "role_play/character", |
| "role_play/scenario", |
| ], |
| "empathy": [ |
| "empathy/anger", |
| "empathy/sadness_disappointment", |
| "empathy/anxiety_fear", |
| "empathy/joy_excitement", |
| ], |
| } |
|
|
| ABILITY_TO_COL = { |
| "acoustic_attributes/age": "age", |
| "acoustic_attributes/speed": "speed", |
| "acoustic_attributes/gender": "gend", |
| "acoustic_attributes/emotion": "emot_ac", |
| "acoustic_attributes/pitch": "pitch", |
| "acoustic_attributes/volume": "vol", |
| "acoustic_attributes/composite_properties": "comp", |
| "instruction/emotion": "emot_in", |
| "instruction/style": "style", |
| "instruction/variation": "vari", |
| "role_play/scenario": "scen", |
| "role_play/character": "char", |
| "empathy/anger": "anger", |
| "empathy/sadness_disappointment": "sad", |
| "empathy/anxiety_fear": "anx", |
| "empathy/joy_excitement": "joy", |
| } |
|
|
| MAJOR_GROUPS = { |
| "acoustic": ["age", "speed", "gend", "emot_ac", "pitch", "vol", "comp"], |
| "instruct": ["emot_in", "style", "vari"], |
| "roleplay": ["scen", "char"], |
| "empathy": ["anger", "sad", "anx", "joy"], |
| } |
| COL_ORDER = ( |
| MAJOR_GROUPS["acoustic"] |
| + MAJOR_GROUPS["instruct"] |
| + MAJOR_GROUPS["roleplay"] |
| + MAJOR_GROUPS["empathy"] |
| ) |
| DEFAULT_MODEL_ROWS = [ |
| ("Qwen2.5-Omni-7B TF", "qwen25omni_transformers", "en"), |
| ("Qwen2.5-Omni-7B TF", "qwen25omni_transformers", "zh"), |
| ("Qwen2.5-Omni-7B vLLM", "qwen25omni_vllm", "en"), |
| ("Qwen2.5-Omni-7B vLLM", "qwen25omni_vllm", "zh"), |
| ("Qwen3-Omni-30B TF", "qwen3omni_transformers", "en"), |
| ("Qwen3-Omni-30B TF", "qwen3omni_transformers", "zh"), |
| ("Qwen3-Omni-30B vLLM", "qwen3omni_vllm", "en"), |
| ("Qwen3-Omni-30B vLLM", "qwen3omni_vllm", "zh"), |
| ] |
|
|
|
|
| def build_weights(): |
| big_cat_w = 0.25 |
| weights = {} |
|
|
| comp = "acoustic_attributes/composite_properties" |
| comp_w = big_cat_w * 0.5 |
| rem_w = (big_cat_w - comp_w) / (len(SUB_CATS["acoustic_attributes"]) - 1) |
| for ability in SUB_CATS["acoustic_attributes"]: |
| weights[ability] = comp_w if ability == comp else rem_w |
|
|
| for big in ("instruction", "role_play", "empathy"): |
| each = big_cat_w / len(SUB_CATS[big]) |
| for ability in SUB_CATS[big]: |
| weights[ability] = each |
|
|
| return weights |
|
|
|
|
| WEIGHTS = build_weights() |
| COL_TO_ABILITY = {v: k for k, v in ABILITY_TO_COL.items()} |
|
|
|
|
| def safe_float(value): |
| try: |
| return float(value) |
| except (TypeError, ValueError): |
| return None |
|
|
|
|
| def load_scores(path: Path): |
| scores = defaultdict(list) |
| with path.open("r", encoding="utf-8") as f: |
| for line in f: |
| line = line.strip() |
| if not line: |
| continue |
| obj = json.loads(line) |
| ability = obj.get("ability", "") |
| col = ABILITY_TO_COL.get(ability) |
| score = safe_float(obj.get("gemini_score")) |
| if col and score is not None: |
| scores[col].append(score) |
| return scores |
|
|
|
|
| def avg(values): |
| return sum(values) / len(values) if values else math.nan |
|
|
|
|
| def weighted_overall(scores): |
| weighted_sum = 0.0 |
| used_w = 0.0 |
| for col, ability in COL_TO_ABILITY.items(): |
| values = scores.get(col, []) |
| if not values: |
| continue |
| w = WEIGHTS[ability] |
| weighted_sum += avg(values) * w |
| used_w += w |
| return weighted_sum / used_w if used_w else math.nan |
|
|
|
|
| def best_per_lang(results, col): |
| best = set() |
| for lang in ("en", "zh"): |
| keys = [k for k in results if k[2] == lang and results[k] is not None] |
| if not keys: |
| continue |
| best.add(max(keys, key=lambda k: results[k][col])) |
| return best |
|
|
|
|
| def fmt(value, key, bold_set): |
| if value is None or math.isnan(value): |
| return "--" |
| out = f"{value:.2f}" |
| return r"\textbf{" + out + "}" if key in bold_set else out |
|
|
|
|
| def parse_rows(row_specs): |
| rows = [] |
| for spec in row_specs: |
| parts = spec.split(":", 2) |
| if len(parts) != 3: |
| raise ValueError(f"Bad --rows item: {spec!r}; expected display:model_key:lang") |
| rows.append(tuple(parts)) |
| return rows |
|
|
|
|
| def make_results(score_root: Path, judge_tag: str, model_rows): |
| results = {} |
| for display_name, model_key, lang in model_rows: |
| path = score_root / judge_tag / model_key / lang / "metadata_with_score.jsonl" |
| key = (display_name, model_key, lang) |
| if not path.exists(): |
| print(f"[WARN] File not found: {path}") |
| results[key] = None |
| continue |
|
|
| scores = load_scores(path) |
| row = {"overall": weighted_overall(scores)} |
| for col in COL_ORDER: |
| row[col] = avg(scores.get(col, [])) |
| results[key] = row |
| n = sum(len(v) for v in scores.values()) |
| print(f"Loaded {display_name} {lang}: overall={row['overall']:.2f}, n={n}") |
| return results |
|
|
|
|
| def render_latex(results, model_rows): |
| header = r"""\documentclass{article} |
| \usepackage{booktabs} |
| \usepackage{multirow} |
| \usepackage{array} |
| \usepackage{geometry} |
| \usepackage{graphicx} |
| |
| \geometry{landscape, left=0.8cm, right=0.8cm, top=1.5cm, bottom=1.5cm} |
| \newcommand{\rh}[1]{\rotatebox{45}{#1}} |
| |
| \begin{document} |
| \begin{table}[ht] |
| \centering |
| \scriptsize |
| \setlength{\tabcolsep}{2.5pt} |
| \renewcommand{\arraystretch}{1.1} |
| \caption{Evaluation results on VStyle. Overall follows the official weighted aggregation.} |
| \label{tab:vstyle_ours} |
| \begin{tabular}{@{} l @{\hspace{3pt}} c @{\hspace{3pt}} c |
| | *{7}{c} |
| | *{3}{c} |
| | *{2}{c} |
| | *{4}{c} @{}} |
| \toprule |
| \multirow{2}{*}{Model} |
| & \multirow{2}{*}{Lang} |
| & \multirow{2}{*}{Overall} |
| & \multicolumn{7}{c|}{Acoustic Attributes} |
| & \multicolumn{3}{c|}{Instruction} |
| & \multicolumn{2}{c|}{Role-Play} |
| & \multicolumn{4}{c}{Empathy} \\ |
| \cmidrule(lr){4-10}\cmidrule(lr){11-13}\cmidrule(lr){14-15}\cmidrule(lr){16-19} |
| & & & |
| \rh{Age} & \rh{Speed} & \rh{Gend.} & \rh{Emot.} & \rh{Pitch} & \rh{Vol.} & \rh{Comp.} & |
| \rh{Emot.} & \rh{Style} & \rh{Vari.} & |
| \rh{Scen.} & \rh{Char.} & |
| \rh{Anger} & \rh{Sad.} & \rh{Anx.} & \rh{Joy} \\ |
| \midrule |
| """ |
| footer = r"""\bottomrule |
| \end{tabular} |
| \end{table} |
| \end{document} |
| """ |
| bold_sets = {col: best_per_lang(results, col) for col in COL_ORDER + ["overall"]} |
| rows = [] |
| previous_model = None |
|
|
| for display_name, model_key, lang in model_rows: |
| key = (display_name, model_key, lang) |
| row = results.get(key) or {col: math.nan for col in COL_ORDER + ["overall"]} |
| model_cell = r"\multirow{2}{*}{" + display_name + "}" if display_name != previous_model else "" |
| previous_model = display_name |
|
|
| cells = [model_cell, lang, fmt(row["overall"], key, bold_sets["overall"])] |
| cells.extend(fmt(row[col], key, bold_sets[col]) for col in COL_ORDER) |
| rows.append(" " + " & ".join(cells) + r" \\") |
|
|
| if lang == "zh" and display_name != model_rows[-1][0]: |
| rows.append(r"\midrule") |
|
|
| return header + "\n".join(rows) + "\n" + footer |
|
|
|
|
| def main(): |
| script_dir = Path(__file__).resolve().parent |
| eval_root = script_dir.parent |
| default_judge = os.environ.get("JUDGE_TAG") or os.environ.get("VSTYLE_JUDGE_MODEL", "gemini-2.5-pro") |
| default_judge = default_judge.replace("/", "_") |
|
|
| parser = argparse.ArgumentParser() |
| parser.add_argument("--score_root", type=Path, default=eval_root / "scoring" / "outputs") |
| parser.add_argument("--judge_tag", default=default_judge) |
| parser.add_argument("--out", type=Path, default=script_dir / "vstyle_table.tex") |
| parser.add_argument( |
| "--rows", |
| nargs="*", |
| default=None, |
| help="Optional rows as display:model_key:lang, e.g. 'Qwen3 vLLM:qwen3omni_vllm:en'", |
| ) |
| args = parser.parse_args() |
|
|
| model_rows = parse_rows(args.rows) if args.rows else DEFAULT_MODEL_ROWS |
| results = make_results(args.score_root, args.judge_tag, model_rows) |
| latex = render_latex(results, model_rows) |
| args.out.parent.mkdir(parents=True, exist_ok=True) |
| args.out.write_text(latex, encoding="utf-8") |
| print(f"\nLaTeX table written to: {args.out.resolve()}") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
