#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Generate a compact VStyle-style LaTeX table from metadata_with_score.jsonl files. Default layout: scoring/outputs////metadata_with_score.jsonl """ import argparse import json import math import os from collections import defaultdict from pathlib import Path SUB_CATS = { "acoustic_attributes": [ "acoustic_attributes/age", "acoustic_attributes/speed", "acoustic_attributes/gender", "acoustic_attributes/emotion", "acoustic_attributes/pitch", "acoustic_attributes/volume", "acoustic_attributes/composite_properties", ], "instruction": [ "instruction/emotion", "instruction/variation", "instruction/style", ], "role_play": [ "role_play/character", "role_play/scenario", ], "empathy": [ "empathy/anger", "empathy/sadness_disappointment", "empathy/anxiety_fear", "empathy/joy_excitement", ], } ABILITY_TO_COL = { "acoustic_attributes/age": "age", "acoustic_attributes/speed": "speed", "acoustic_attributes/gender": "gend", "acoustic_attributes/emotion": "emot_ac", "acoustic_attributes/pitch": "pitch", "acoustic_attributes/volume": "vol", "acoustic_attributes/composite_properties": "comp", "instruction/emotion": "emot_in", "instruction/style": "style", "instruction/variation": "vari", "role_play/scenario": "scen", "role_play/character": "char", "empathy/anger": "anger", "empathy/sadness_disappointment": "sad", "empathy/anxiety_fear": "anx", "empathy/joy_excitement": "joy", } MAJOR_GROUPS = { "acoustic": ["age", "speed", "gend", "emot_ac", "pitch", "vol", "comp"], "instruct": ["emot_in", "style", "vari"], "roleplay": ["scen", "char"], "empathy": ["anger", "sad", "anx", "joy"], } COL_ORDER = ( MAJOR_GROUPS["acoustic"] + MAJOR_GROUPS["instruct"] + MAJOR_GROUPS["roleplay"] + MAJOR_GROUPS["empathy"] ) DEFAULT_MODEL_ROWS = [ ("Qwen2.5-Omni-7B TF", "qwen25omni_transformers", "en"), ("Qwen2.5-Omni-7B TF", "qwen25omni_transformers", "zh"), ("Qwen2.5-Omni-7B vLLM", "qwen25omni_vllm", "en"), ("Qwen2.5-Omni-7B vLLM", "qwen25omni_vllm", "zh"), ("Qwen3-Omni-30B TF", "qwen3omni_transformers", "en"), ("Qwen3-Omni-30B TF", "qwen3omni_transformers", "zh"), ("Qwen3-Omni-30B vLLM", "qwen3omni_vllm", "en"), ("Qwen3-Omni-30B vLLM", "qwen3omni_vllm", "zh"), ] def build_weights(): big_cat_w = 0.25 weights = {} comp = "acoustic_attributes/composite_properties" comp_w = big_cat_w * 0.5 rem_w = (big_cat_w - comp_w) / (len(SUB_CATS["acoustic_attributes"]) - 1) for ability in SUB_CATS["acoustic_attributes"]: weights[ability] = comp_w if ability == comp else rem_w for big in ("instruction", "role_play", "empathy"): each = big_cat_w / len(SUB_CATS[big]) for ability in SUB_CATS[big]: weights[ability] = each return weights WEIGHTS = build_weights() COL_TO_ABILITY = {v: k for k, v in ABILITY_TO_COL.items()} def safe_float(value): try: return float(value) except (TypeError, ValueError): return None def load_scores(path: Path): scores = defaultdict(list) with path.open("r", encoding="utf-8") as f: for line in f: line = line.strip() if not line: continue obj = json.loads(line) ability = obj.get("ability", "") col = ABILITY_TO_COL.get(ability) score = safe_float(obj.get("gemini_score")) if col and score is not None: scores[col].append(score) return scores def avg(values): return sum(values) / len(values) if values else math.nan def weighted_overall(scores): weighted_sum = 0.0 used_w = 0.0 for col, ability in COL_TO_ABILITY.items(): values = scores.get(col, []) if not values: continue w = WEIGHTS[ability] weighted_sum += avg(values) * w used_w += w return weighted_sum / used_w if used_w else math.nan def best_per_lang(results, col): best = set() for lang in ("en", "zh"): keys = [k for k in results if k[2] == lang and results[k] is not None] if not keys: continue best.add(max(keys, key=lambda k: results[k][col])) return best def fmt(value, key, bold_set): if value is None or math.isnan(value): return "--" out = f"{value:.2f}" return r"\textbf{" + out + "}" if key in bold_set else out def parse_rows(row_specs): rows = [] for spec in row_specs: parts = spec.split(":", 2) if len(parts) != 3: raise ValueError(f"Bad --rows item: {spec!r}; expected display:model_key:lang") rows.append(tuple(parts)) return rows def make_results(score_root: Path, judge_tag: str, model_rows): results = {} for display_name, model_key, lang in model_rows: path = score_root / judge_tag / model_key / lang / "metadata_with_score.jsonl" key = (display_name, model_key, lang) if not path.exists(): print(f"[WARN] File not found: {path}") results[key] = None continue scores = load_scores(path) row = {"overall": weighted_overall(scores)} for col in COL_ORDER: row[col] = avg(scores.get(col, [])) results[key] = row n = sum(len(v) for v in scores.values()) print(f"Loaded {display_name} {lang}: overall={row['overall']:.2f}, n={n}") return results def render_latex(results, model_rows): header = r"""\documentclass{article} \usepackage{booktabs} \usepackage{multirow} \usepackage{array} \usepackage{geometry} \usepackage{graphicx} \geometry{landscape, left=0.8cm, right=0.8cm, top=1.5cm, bottom=1.5cm} \newcommand{\rh}[1]{\rotatebox{45}{#1}} \begin{document} \begin{table}[ht] \centering \scriptsize \setlength{\tabcolsep}{2.5pt} \renewcommand{\arraystretch}{1.1} \caption{Evaluation results on VStyle. Overall follows the official weighted aggregation.} \label{tab:vstyle_ours} \begin{tabular}{@{} l @{\hspace{3pt}} c @{\hspace{3pt}} c | *{7}{c} | *{3}{c} | *{2}{c} | *{4}{c} @{}} \toprule \multirow{2}{*}{Model} & \multirow{2}{*}{Lang} & \multirow{2}{*}{Overall} & \multicolumn{7}{c|}{Acoustic Attributes} & \multicolumn{3}{c|}{Instruction} & \multicolumn{2}{c|}{Role-Play} & \multicolumn{4}{c}{Empathy} \\ \cmidrule(lr){4-10}\cmidrule(lr){11-13}\cmidrule(lr){14-15}\cmidrule(lr){16-19} & & & \rh{Age} & \rh{Speed} & \rh{Gend.} & \rh{Emot.} & \rh{Pitch} & \rh{Vol.} & \rh{Comp.} & \rh{Emot.} & \rh{Style} & \rh{Vari.} & \rh{Scen.} & \rh{Char.} & \rh{Anger} & \rh{Sad.} & \rh{Anx.} & \rh{Joy} \\ \midrule """ footer = r"""\bottomrule \end{tabular} \end{table} \end{document} """ bold_sets = {col: best_per_lang(results, col) for col in COL_ORDER + ["overall"]} rows = [] previous_model = None for display_name, model_key, lang in model_rows: key = (display_name, model_key, lang) row = results.get(key) or {col: math.nan for col in COL_ORDER + ["overall"]} model_cell = r"\multirow{2}{*}{" + display_name + "}" if display_name != previous_model else "" previous_model = display_name cells = [model_cell, lang, fmt(row["overall"], key, bold_sets["overall"])] cells.extend(fmt(row[col], key, bold_sets[col]) for col in COL_ORDER) rows.append(" " + " & ".join(cells) + r" \\") if lang == "zh" and display_name != model_rows[-1][0]: rows.append(r"\midrule") return header + "\n".join(rows) + "\n" + footer def main(): script_dir = Path(__file__).resolve().parent eval_root = script_dir.parent default_judge = os.environ.get("JUDGE_TAG") or os.environ.get("VSTYLE_JUDGE_MODEL", "gemini-2.5-pro") default_judge = default_judge.replace("/", "_") parser = argparse.ArgumentParser() parser.add_argument("--score_root", type=Path, default=eval_root / "scoring" / "outputs") parser.add_argument("--judge_tag", default=default_judge) parser.add_argument("--out", type=Path, default=script_dir / "vstyle_table.tex") parser.add_argument( "--rows", nargs="*", default=None, help="Optional rows as display:model_key:lang, e.g. 'Qwen3 vLLM:qwen3omni_vllm:en'", ) args = parser.parse_args() model_rows = parse_rows(args.rows) if args.rows else DEFAULT_MODEL_ROWS results = make_results(args.score_root, args.judge_tag, model_rows) latex = render_latex(results, model_rows) args.out.parent.mkdir(parents=True, exist_ok=True) args.out.write_text(latex, encoding="utf-8") print(f"\nLaTeX table written to: {args.out.resolve()}") if __name__ == "__main__": main()