Spaces:
Sleeping
Sleeping
File size: 2,538 Bytes
8512b02 7c4db09 8512b02 7c4db09 8512b02 7c4db09 8512b02 7c4db09 8512b02 7c4db09 8512b02 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | from __future__ import annotations
import json
from math import sqrt
from statistics import mean, pstdev
from typing import Dict, List
from app.main import _rollout_episode
from app.env import TASKS
def _stats(scores: List[float]) -> Dict[str, float]:
if not scores:
return {"avg": 0.0, "std": 0.0, "ci95_low": 0.0, "ci95_high": 0.0}
avg = mean(scores)
std = pstdev(scores) if len(scores) > 1 else 0.0
margin = 1.96 * (std / sqrt(max(1, len(scores))))
return {
"avg": round(avg, 6),
"std": round(std, 6),
"ci95_low": round(max(0.0, avg - margin), 6),
"ci95_high": round(min(1.0, avg + margin), 6),
}
def _mean_diff_z(a: List[float], b: List[float]) -> float:
if not a or not b:
return 0.0
n = min(len(a), len(b))
paired = [a[i] - b[i] for i in range(n)]
avg = mean(paired)
std = pstdev(paired) if len(paired) > 1 else 0.0
if std == 0.0:
return 0.0
return round(avg / (std / sqrt(max(1, len(paired)))), 6)
def main() -> None:
seeds = list(range(42, 72))
policies = ["noop", "random-safe", "baseline", "reasoning", "trained"]
report: Dict[str, object] = {
"seeds": seeds,
"policies": policies,
"tasks": {},
}
for task_id in TASKS:
task_scores: Dict[str, List[float]] = {policy: [] for policy in policies}
for seed in seeds:
for policy in policies:
rollout = _rollout_episode(task_id=task_id, seed=seed, policy=policy)
task_scores[policy].append(float(rollout["score"]))
report["tasks"][task_id] = {
"noop": _stats(task_scores["noop"]),
"random-safe": _stats(task_scores["random-safe"]),
"baseline": _stats(task_scores["baseline"]),
"reasoning": _stats(task_scores["reasoning"]),
"trained": _stats(task_scores["trained"]),
"reasoning_minus_baseline": round(mean(task_scores["reasoning"]) - mean(task_scores["baseline"]), 6),
"baseline_minus_noop": round(mean(task_scores["baseline"]) - mean(task_scores["noop"]), 6),
"trained_minus_reasoning": round(mean(task_scores["trained"]) - mean(task_scores["reasoning"]), 6),
"z_reasoning_vs_baseline": _mean_diff_z(task_scores["reasoning"], task_scores["baseline"]),
"z_trained_vs_reasoning": _mean_diff_z(task_scores["trained"], task_scores["reasoning"]),
}
print(json.dumps(report, indent=2))
if __name__ == "__main__":
main()
|