Spaces:
Running
Running
File size: 5,008 Bytes
e25024e 45113e6 e25024e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 | #!/usr/bin/env python3
"""Aggregate community contributions into paper-ready tables.
Usage:
python scripts/aggregate_contributions.py [--dir community_results] [--format latex|csv|json]
Reads all contribution JSON files from the specified directory, aggregates
them by model and method, and outputs summary tables suitable for inclusion
in the paper.
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
# Add project root to path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from obliteratus.community import (
aggregate_results,
generate_latex_table,
load_contributions,
)
def main():
parser = argparse.ArgumentParser(
description="Aggregate community contributions into paper tables."
)
parser.add_argument(
"--dir",
default="community_results",
help="Directory containing contribution JSON files (default: community_results)",
)
parser.add_argument(
"--format",
choices=["latex", "csv", "json", "summary"],
default="summary",
help="Output format (default: summary)",
)
parser.add_argument(
"--metric",
default="refusal_rate",
help="Metric to display in tables (default: refusal_rate)",
)
parser.add_argument(
"--methods",
nargs="*",
help="Methods to include (default: all)",
)
parser.add_argument(
"--min-runs",
type=int,
default=1,
help="Minimum runs per (model, method) to include (default: 1)",
)
args = parser.parse_args()
# Load all contributions
records = load_contributions(args.dir)
if not records:
print(f"No contributions found in {args.dir}/", file=sys.stderr)
sys.exit(1)
print(f"Loaded {len(records)} contribution(s) from {args.dir}/", file=sys.stderr)
# Aggregate
aggregated = aggregate_results(records)
# Filter by minimum runs
if args.min_runs > 1:
for model in list(aggregated.keys()):
for method in list(aggregated[model].keys()):
if aggregated[model][method]["n_runs"] < args.min_runs:
del aggregated[model][method]
if not aggregated[model]:
del aggregated[model]
if not aggregated:
print("No results meet the minimum run threshold.", file=sys.stderr)
sys.exit(1)
# Output
if args.format == "summary":
_print_summary(aggregated, args.metric)
elif args.format == "latex":
print(generate_latex_table(aggregated, methods=args.methods, metric=args.metric))
elif args.format == "json":
print(json.dumps(aggregated, indent=2))
elif args.format == "csv":
_print_csv(aggregated, args.metric)
def _print_summary(aggregated: dict, metric: str):
"""Print a human-readable summary of aggregated results."""
total_runs = sum(
data["n_runs"]
for model_data in aggregated.values()
for data in model_data.values()
)
n_models = len(aggregated)
n_methods = len(set(
method
for model_data in aggregated.values()
for method in model_data
))
print(f"\n{'=' * 70}")
print("Community Contribution Summary")
print(f"{'=' * 70}")
print(f" Total runs: {total_runs}")
print(f" Models: {n_models}")
print(f" Methods: {n_methods}")
print()
for model in sorted(aggregated.keys()):
model_data = aggregated[model]
short = model.split("/")[-1] if "/" in model else model
print(f" {short}:")
for method in sorted(model_data.keys()):
data = model_data[method]
n = data["n_runs"]
if metric in data:
stats = data[metric]
mean = stats["mean"]
std = stats["std"]
if std > 0 and n > 1:
print(f" {method:20s} {metric}={mean:.2f} ± {std:.2f} (n={n})")
else:
print(f" {method:20s} {metric}={mean:.2f} (n={n})")
else:
print(f" {method:20s} (no {metric} data, n={n})")
print()
print(f"{'=' * 70}")
print(f"To generate LaTeX: python {sys.argv[0]} --format latex")
print(f"To generate CSV: python {sys.argv[0]} --format csv")
def _print_csv(aggregated: dict, metric: str):
"""Print results as CSV."""
print("model,method,n_runs,mean,std,min,max")
for model in sorted(aggregated.keys()):
for method in sorted(aggregated[model].keys()):
data = aggregated[model][method]
n = data["n_runs"]
if metric in data:
stats = data[metric]
print(
f"{model},{method},{n},"
f"{stats['mean']:.4f},{stats['std']:.4f},"
f"{stats['min']:.4f},{stats['max']:.4f}"
)
if __name__ == "__main__":
main()
|