scripts/claim1_accuracy.py

#!/usr/bin/env python3
"""Claim 1 - HybridFlow accuracy (Table 1).

HONEST SCOPE: the full accuracy table requires paid GPT-4.1 + a local
Llama3.2-3B (vLLM) run over four benchmarks (GPQA/MMLU-Pro/AIME24/LiveBench),
which is NOT re-run here (no API calls). What we DO verify on CPU is that the
paper's reported per-benchmark cells are INTERNALLY CONSISTENT with the reported
averages, and we record the paper's own open-model target (Table 8) that a full
open replication should hit.

Writes outputs/claim1.json.
"""
import json
import os
from statistics import mean

import matplotlib

matplotlib.use("Agg")
import matplotlib.pyplot as plt

HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
OUT = os.path.join(HERE, "outputs")
FIG = os.path.join(HERE, "figs")
os.makedirs(OUT, exist_ok=True)
os.makedirs(FIG, exist_ok=True)

# Table 1 (accuracy %, per benchmark: GPQA, MMLU-Pro, AIME24, LiveBench-Reasoning)
ROWS = {
    "HybridFlow (Ours)": ([53.33, 72.54, 36.67, 58.83], 55.34),
    "CoT + GPT-4.1 (best non-prompt)": ([57.28, 72.0, 44.42, 62.25], 58.99),
    "Direct Prompt GPT-4.1": ([51.79, 65.5, 37.78, 58.25], 53.33),
    "HybridLLM": ([52.9, 43.0, 22.22, 36.67], 38.70),
    "DoT": ([50.54, 66.0, 21.11, 48.33], 46.50),
}

print("Claim 1: Table 1 accuracy internal-consistency (avg == mean of cells)")
print(f"{'Method':<34}{'mean(cells)':>12}{'reported':>10}  ok")
checks = []
all_ok = True
for name, (cells, reported) in ROWS.items():
    m = mean(cells)
    ok = abs(round(m, 2) - reported) <= 0.011
    all_ok = all_ok and ok
    assert ok, f"{name}: mean {m:.4f} != reported {reported}"
    print(f"{name:<34}{m:>12.4f}{reported:>10.2f}  {'OK' if ok else 'FAIL'}")
    checks.append(
        {
            "method": name,
            "cells": cells,
            "mean_computed": round(m, 4),
            "reported_avg": reported,
            "match": ok,
        }
    )

# HybridFlow (55.34) vs best non-prompt baseline CoT+GPT-4.1 (58.99): gap 3.65 pts,
# but HybridFlow runs under edge-cloud collaboration at far lower cloud cost (Claim 2).
gap = 58.99 - 55.34
print(
    f"\nHybridFlow 55.34% vs best non-prompt CoT+GPT-4.1 58.99% -> gap {gap:.2f} pts "
    f"(HybridFlow uses far less cloud; see Claim 2)."
)

# Paper's own open-model swap (Table 8, GPQA): a documented open replication target.
table8 = {
    "note": "GPQA, swapped pair: Qwen2.5-7B edge + DeepSeek-V3 cloud (open models)",
    "All-Edge CoT (Qwen2.5-7B)": {
        "acc_pct": 34,
        "api_cost_e3_usd": None,
        "latency_s": 19.52,
    },
    "All-Cloud CoT (DeepSeek-V3)": {
        "acc_pct": 59,
        "api_cost_e3_usd": 6.70,
        "latency_s": 61.00,
    },
    "HybridFlow (Ours)": {"acc_pct": 53, "api_cost_e3_usd": 1.16, "latency_s": 36.86},
}
print(
    "\nOpen-model replication target (Table 8): HybridFlow 53% acc, 1.16e-3 $, 36.86 s."
)

# Figure: reported average accuracy per method (internal-consistency values).
fig, ax = plt.subplots(figsize=(8.0, 4.2))
names = list(ROWS.keys())
avgs = [ROWS[n][1] for n in names]
cols = ["#d1495b" if n.startswith("HybridFlow") else "#888888" for n in names]
bars = ax.bar([n.replace(" (best non-prompt)", "") for n in names], avgs, color=cols)
for b, v in zip(bars, avgs):
    ax.text(b.get_x() + b.get_width() / 2, v + 0.4, f"{v:.2f}", ha="center", fontsize=8)
ax.axhline(58.99, ls="--", lw=0.8, color="#0b6e4f", label="best non-prompt (58.99)")
ax.set_ylabel("Avg accuracy % (Table 1)")
ax.set_title("Claim 1: reported avg accuracy (internal-consistency; not a re-run)")
ax.tick_params(axis="x", rotation=18, labelsize=8)
ax.legend()
plt.tight_layout()
fig.savefig(os.path.join(FIG, "claim1_accuracy.png"), dpi=130)
plt.close(fig)

result = {
    "claim": "1 - Table 1 accuracy (HybridFlow 55.34% avg vs 58.99% CoT+GPT-4.1)",
    "scope": "INTERNAL-CONSISTENCY ONLY (no API/GPU re-run). Full table needs paid "
    "GPT-4.1 + local Llama3.2-3B over 4 benchmarks.",
    "consistency_checks": checks,
    "all_consistent": all_ok,
    "hybridflow_vs_best_nonprompt_gap_pts": round(gap, 2),
    "open_model_target_table8": table8,
    "figure": "figs/claim1_accuracy.png",
    "verdict": "PASS (internal consistency): reported averages equal the mean of the "
    "reported per-benchmark cells for every method. Absolute accuracy NOT "
    "re-run (needs paid API). Documented open target: 53% acc @ Table 8.",
}
with open(os.path.join(OUT, "claim1.json"), "w") as f:
    json.dump(result, f, indent=2)
print(f"\nAll consistency checks pass: {all_ok}")
print("VERDICT: PASS (internal consistency); absolute accuracy needs paid re-run.")