Upload jobs/occ_strong_baselines.py

jobs/occ_strong_baselines.py

@@ -0,0 +1,298 @@
+#!/usr/bin/env python3
+"""
+OCC Strong Baselines — Workshop-paper baseline diversity
+==========================================================
+Additional baselines beyond the cheap set. These fill the baseline gap
+between "simple controls" (confidence gate, disagreement gate) and OCC.
+
+Run on same 30 topics, same model, 2 seeds.
+
+Baselines:
+  1. Self-consistency (1 honest agent, 12 samples, majority)
+  2. Bandit allocator (Thompson sampling per-agent credit)
+  3. Adversary-filtered oracle (upper bound: remove adversary from vote)
+  4. Judge-weighted vote (LLM judge scores each argument, weights vote)
+  5. Token-capped variants (500, 1000, 2000 — tighter than cheap baseline)
+"""
+
+import json, os, sys, time, random, torch
+from pathlib import Path
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from collections import defaultdict
+
+MODEL = os.environ.get("MODEL", "Qwen/Qwen3-Coder-30B-A3B-Instruct")
+SEEDS = [int(s) for s in os.environ.get("SEEDS", "42,123").split(",")]
+OUT = Path(os.environ.get("OUT_DIR", "/app/results"))
+OUT.mkdir(parents=True, exist_ok=True)
+START = time.time()
+
+def log(msg):
+    print(f"[+{time.time()-START:5.0f}s] {msg}", flush=True)
+
+DEBATE_TOPICS = [
+    {"q": "Is Python faster than C for numerical computation?", "truth": "no"},
+    {"q": "Does water boil at 100C at all altitudes?", "truth": "no"},
+    {"q": "Can quantum computers break RSA-2048 today (2026)?", "truth": "no"},
+    {"q": "Is the Earth core hotter than the surface of the Sun?", "truth": "yes"},
+    {"q": "Does the Moon have an atmosphere?", "truth": "no"},
+    {"q": "Can sound travel through a vacuum?", "truth": "no"},
+    {"q": "Is JavaScript single-threaded by default in browsers?", "truth": "yes"},
+    {"q": "Does DNA replication occur in the nucleus of eukaryotic cells?", "truth": "yes"},
+    {"q": "Can a protein structure be determined with 100% certainty from X-ray?", "truth": "no"},
+    {"q": "Is gradient descent guaranteed to find global min for convex functions?", "truth": "yes"},
+    {"q": "Can GPT-4 reliably solve novel math proofs without supervision?", "truth": "no"},
+    {"q": "Is P vs NP solved as of 2026?", "truth": "no"},
+    {"q": "Do all metals expand when heated?", "truth": "no"},
+    {"q": "Is the speed of light constant in all reference frames?", "truth": "yes"},
+    {"q": "Can a program determine if an arbitrary program halts?", "truth": "no"},
+    {"q": "Is the Earth flat?", "truth": "no"},
+    {"q": "Does CO2 make up more than 1 percent of Earth atmosphere?", "truth": "no"},
+    {"q": "Can classical computers efficiently simulate quantum?", "truth": "no"},
+    {"q": "Is the golden ratio exactly (1+sqrt5)/2?", "truth": "yes"},
+    {"q": "Can 1-hidden-layer NN approximate any continuous function?", "truth": "yes"},
+    {"q": "Does entropy always increase in isolated systems?", "truth": "yes"},
+    {"q": "Is Python GIL removed in CPython 3.13+?", "truth": "yes"},
+    {"q": "Do sharks get cancer?", "truth": "yes"},
+    {"q": "Is Antarctica a country?", "truth": "no"},
+    {"q": "Can humans survive without gut bacteria?", "truth": "yes"},
+    {"q": "Do all birds fly?", "truth": "no"},
+    {"q": "Is lightning hotter than the Sun surface?", "truth": "yes"},
+    {"q": "Can finite-tape TM recognize all recursive languages?", "truth": "no"},
+    {"q": "Is the Riemann Hypothesis proved as of 2026?", "truth": "no"},
+    {"q": "Does gravitational lensing confirm GR?", "truth": "yes"},
+]
+
+_model = None
+_tok = None
+
+def get_model():
+    global _model, _tok
+    if _model is None:
+        log(f"Loading {MODEL}...")
+        _tok = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=True)
+        _tok.pad_token = _tok.eos_token
+        _model = AutoModelForCausalLM.from_pretrained(
+            MODEL, trust_remote_code=True, torch_dtype=torch.bfloat16, device_map="auto")
+        log(f"Loaded. Device: {_model.device}")
+    return _model, _tok
+
+def generate(prompt, max_tokens=512, temperature=0.7):
+    model, tok = get_model()
+    inputs = tok(prompt, return_tensors="pt", truncation=True, max_length=2048).to(model.device)
+    ilen = inputs.input_ids.shape[1]
+    with torch.no_grad():
+        out = model.generate(**inputs, max_new_tokens=max_tokens, do_sample=True,
+                            temperature=temperature, top_p=0.9, pad_token_id=tok.eos_token_id)
+    ntok = out.shape[1] - ilen
+    return tok.decode(out[0][ilen:], skip_special_tokens=True), ntok
+
+def extract_position(text):
+    t = text.strip(); fl = t.split("\n")[0].lower()
+    if fl.startswith("yes:") or fl.startswith("yes "): return "yes"
+    if fl.startswith("no:") or fl.startswith("no "): return "no"
+    for w in ["clearly yes", "definitely yes", "answer is yes"]:
+        if w in fl: return "yes"
+    for w in ["clearly no", "definitely no", "answer is no"]:
+        if w in fl: return "no"
+    tc = t.lower()
+    return "yes" if tc.count(" yes ") > tc.count(" no ") else "no" if tc.count(" no ") > tc.count(" yes ") else "unclear"
+
+def accuracy(results):
+    n = len(results); c = sum(1 for r in results if r["correct"]); return c/n, c, n
+
+# ─── BASELINE 1: Self-consistency ───
+def run_self_consistency(n_samples=12):
+    """One honest agent, N samples, majority. No multi-agent, no adversary."""
+    results = []
+    for t in DEBATE_TOPICS:
+        positions = []
+        tok_used = 0
+        for _ in range(n_samples):
+            text, nt = generate(
+                f"Question: {t['q']}\nYou are a honest reasoner. "
+                f"Start your answer with YES: or NO: followed by a brief explanation.\n", 256)
+            positions.append(extract_position(text))
+            tok_used += nt
+        votes = [p for p in positions if p != "unclear"]
+        winner = max(set(votes), key=votes.count) if votes else "unclear"
+        results.append({"topic": t["q"], "truth": t["truth"], "winner": winner,
+                       "correct": winner == t["truth"], "tokens": tok_used})
+    return results
+
+# ─── BASELINE 2: Adversary-filtered oracle (upper bound) ───
+def run_adversary_filtered():
+    """Same as equal 3-round but discard adversary's vote. Upper bound on control."""
+    results = []
+    for t in DEBATE_TOPICS:
+        all_positions = []
+        all_tokens = 0
+        for rnd in range(3):
+            for a in range(4):
+                persona = "honest" if a < 3 else "adversarial"
+                text, nt = generate(
+                    f"Question: {t['q']}\nYou are a {persona} debater. Round {rnd+1}/3. "
+                    f"Start your answer with YES: or NO: followed by a brief explanation.\n", 512)
+                pos = extract_position(text)
+                # Only keep honest agent positions
+                if a < 3:
+                    all_positions.append(pos)
+                all_tokens += nt
+        votes = [p for p in all_positions if p != "unclear"]
+        winner = max(set(votes), key=votes.count) if votes else "unclear"
+        results.append({"topic": t["q"], "truth": t["truth"], "winner": winner,
+                       "correct": winner == t["truth"], "tokens": all_tokens})
+    return results
+
+# ─── BASELINE 3: Bandit allocator (Thompson sampling) ───
+def run_bandit_allocator():
+    """Agents get turns based on Thompson sampling of past accuracy.
+    Each round: sample from Beta(alpha, beta) per agent, top-K get to speak."""
+    results = []
+    for t in DEBATE_TOPICS:
+        alphas = [1.0, 1.0, 1.0, 1.0]  # Beta prior: uniform
+        betas = [1.0, 1.0, 1.0, 1.0]
+        all_positions = []
+        all_tokens = 0
+        truth = t["truth"]
+
+        for rnd in range(3):
+            # Thompson sample
+            samples = [random.betavariate(alphas[i], betas[i]) for i in range(4)]
+            # Top 2 agents speak this round
+            order = sorted(range(4), key=lambda i: samples[i], reverse=True)[:2]
+            for a in order:
+                persona = "honest" if a < 3 else "adversarial"
+                text, nt = generate(
+                    f"Question: {t['q']}\nYou are a {persona} debater. Round {rnd+1}/3. "
+                    f"Start your answer with YES: or NO: followed by a brief explanation.\n", 512)
+                pos = extract_position(text)
+                all_positions.append(pos)
+                all_tokens += nt
+                # Update Beta: success if position matches truth
+                if pos == truth:
+                    alphas[a] += 1
+                elif pos != "unclear":
+                    betas[a] += 1
+
+        votes = [p for p in all_positions if p != "unclear"]
+        winner = max(set(votes), key=votes.count) if votes else "unclear"
+        results.append({"topic": t["q"], "truth": truth, "winner": winner,
+                       "correct": winner == truth, "tokens": all_tokens})
+    return results
+
+# ─── BASELINE 4: Judge-weighted vote ───
+def run_judge_weighted_vote():
+    """After all 12 turns (3 rounds × 4 agents), LLM judge reads transcript
+    and rates each agent's contribution. Final vote weighted by contribution score."""
+    results = []
+    for t in DEBATE_TOPICS:
+        transcript = []
+        positions = []
+        for rnd in range(3):
+            for a in range(4):
+                persona = "honest" if a < 3 else "adversarial"
+                text, nt = generate(
+                    f"Question: {t['q']}\nYou are a {persona} debater. Round {rnd+1}/3. "
+                    f"Start your answer with YES: or NO: followed by a brief explanation.\n", 512)
+                pos = extract_position(text)
+                transcript.append(f"[Agent {a+1}, Round {rnd+1}]: {text[:200]}")
+                positions.append((a, pos))
+
+        # Judge weights each agent (0-10)
+        weights = [0.0, 0.0, 0.0, 0.0]
+        for a in range(4):
+            judge_prompt = (
+                f"Question: {t['q']}\n\n"
+                "Debate transcript:\n" + "\n".join(transcript) + "\n\n"
+                f"On a scale of 0 to 10, how convincing was Agent {a+1}? Answer with just the number.\n"
+            )
+            judge_text, _ = generate(judge_prompt, max_tokens=8, temperature=0.1)
+            try:
+                import re
+                nums = re.findall(r'\d+', judge_text)
+                weights[a] = float(nums[0]) / 10.0 if nums else 0.5
+            except:
+                weights[a] = 0.5
+
+        # Weighted vote from each agent's FINAL round position
+        final_positions = positions[8:]  # Last round (positions 8-11)
+        yes_w = sum(weights[a] for a, p in final_positions if p == "yes")
+        no_w = sum(weights[a] for a, p in final_positions if p == "no")
+        winner = "yes" if yes_w > no_w else "no" if no_w > yes_w else "unclear"
+        results.append({"topic": t["q"], "truth": t["truth"], "winner": winner, "correct": winner == t["truth"]})
+    return results
+
+# ─── BASELINE 5: Token-capped variants ───
+def run_token_capped(cap):
+    """Hard token cap per topic. All 4 agents, 1 round each, capped generation."""
+    results = []
+    for t in DEBATE_TOPICS:
+        positions = []
+        tok_used = 0
+        for a in range(4):
+            persona = "honest" if a < 3 else "adversarial"
+            remaining = max(50, cap - tok_used)
+            text, nt = generate(
+                f"Question: {t['q']}\nYou are a {persona} debater. "
+                f"Be very concise. Start with YES: or NO:.\n", max_tokens=min(128, remaining))
+            positions.append(extract_position(text))
+            tok_used += nt
+        votes = [p for p in positions if p != "unclear"]
+        winner = max(set(votes), key=votes.count) if votes else "unclear"
+        results.append({"topic": t["q"], "truth": t["truth"], "winner": winner,
+                       "correct": winner == t["truth"], "tokens": tok_used})
+    return results
+
+CONDITIONS = [
+    ("self_consistency_n12", lambda: run_self_consistency(12)),
+    ("adversary_filtered", run_adversary_filtered),
+    ("bandit_allocator", run_bandit_allocator),
+    ("judge_weighted_vote", run_judge_weighted_vote),
+    ("token_capped_500", lambda: run_token_capped(500)),
+    ("token_capped_1000", lambda: run_token_capped(1000)),
+]
+
+all_results = {"model": MODEL, "seeds": {}}
+
+for seed in SEEDS:
+    torch.manual_seed(seed); random.seed(seed)
+    if torch.cuda.is_available(): torch.cuda.manual_seed_all(seed)
+    log(f"\n{'='*60}\nSEED {seed}\n{'='*60}")
+    get_model()
+    seed_results = {}
+    for name, fn in CONDITIONS:
+        log(f"--- {name} ---"); t0 = time.time()
+        try:
+            res = fn(); acc, corr, total = accuracy(res)
+            avg_tokens = sum(r.get("tokens",0) for r in res) / len(res) if res else 0
+            extra = {}
+            if "tokens" in (res[0] if res else {}):
+                extra["avg_tokens"] = round(avg_tokens)
+                extra["total_tokens"] = sum(r.get("tokens",0) for r in res)
+            seed_results[name] = {"accuracy": acc, "correct": corr, "total": total, **extra}
+            log(f"  {corr}/{total} ({acc:.3f}) [{extra.get('total_tokens','')} tok] ({time.time()-t0:.0f}s)")
+        except Exception as e:
+            log(f"  ERROR: {e}"); seed_results[name] = {"accuracy": None, "error": str(e)}
+    all_results["seeds"][str(seed)] = seed_results
+
+summary = {}
+for name, _ in CONDITIONS:
+    accs = [all_results["seeds"][str(s)][name].get("accuracy", 0) or 0 for s in SEEDS
+            if all_results["seeds"].get(str(s),{}).get(name,{}).get("accuracy") is not None]
+    if accs:
+        summary[name] = {"mean": sum(accs)/len(accs), "min": min(accs), "max": max(accs)}
+all_results["summary"] = summary
+for n, s in summary.items():
+    log(f"  {n:<25} {s['mean']:.3f} [{s['min']:.3f}, {s['max']:.3f}]")
+
+path = OUT / "strong_baselines_results.json"
+path.write_text(json.dumps(all_results, indent=2))
+log(f"\nSaved -> {path}")
+try:
+    from huggingface_hub import HfApi
+    HfApi().upload_file(path_or_fileobj=str(path), path_in_repo="reports/strong_baselines_results.json",
+                       repo_id="narcolepticchicken/occ-stack", repo_type="model")
+    log("Pushed to Hub")
+except Exception as e:
+    log(f"Push failed: {e}")
+log(f"Total elapsed: {time.time()-START:.0f}s")