Upload jobs/occ_debate_extended_baselines.py

jobs/occ_debate_extended_baselines.py

@@ -0,0 +1,238 @@
+#!/usr/bin/env python3
+"""OCC Debate — Iso-Round + Random-Drop Baselines. H200 run. 3 seeds × 6 conditions.
+FIXED: run_random_drop now returns 4 values (consistent with run_occ)."""
+
+import json, os, sys, time, random, torch
+from pathlib import Path
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+MODEL = os.environ.get("MODEL", "Qwen/Qwen3-Coder-30B-A3B-Instruct")
+SEEDS = [int(s) for s in os.environ.get("SEEDS", "42,123,456").split(",")]
+OUT = Path(os.environ.get("OUT_DIR", "/app/results"))
+OUT.mkdir(parents=True, exist_ok=True)
+START = time.time()
+
+def log(msg):
+    print(f"[+{time.time()-START:5.0f}s] {msg}", flush=True)
+
+DEBATE_TOPICS = [
+    {"q": "Is Python faster than C for numerical computation?", "truth": "no"},
+    {"q": "Does water boil at 100°C at all altitudes?", "truth": "no"},
+    {"q": "Can quantum computers break RSA-2048 today (2026)?", "truth": "no"},
+    {"q": "Is the Earth's core hotter than the surface of the Sun?", "truth": "yes"},
+    {"q": "Does the Moon have an atmosphere?", "truth": "no"},
+    {"q": "Can sound travel through a vacuum?", "truth": "no"},
+    {"q": "Is JavaScript single-threaded by default in browsers?", "truth": "yes"},
+    {"q": "Does DNA replication occur in the nucleus of eukaryotic cells?", "truth": "yes"},
+    {"q": "Can a protein structure be determined with 100% certainty from X-ray?", "truth": "no"},
+    {"q": "Is gradient descent guaranteed to find global min for convex functions?", "truth": "yes"},
+    {"q": "Can GPT-4 reliably solve novel math proofs without supervision?", "truth": "no"},
+    {"q": "Is P vs NP solved as of 2026?", "truth": "no"},
+    {"q": "Do all metals expand when heated?", "truth": "no"},
+    {"q": "Is the speed of light constant in all reference frames?", "truth": "yes"},
+    {"q": "Can a program determine if an arbitrary program halts?", "truth": "no"},
+    {"q": "Is the Earth flat?", "truth": "no"},
+    {"q": "Does CO2 make up >1% of Earth's atmosphere?", "truth": "no"},
+    {"q": "Can classical computers efficiently simulate quantum?", "truth": "no"},
+    {"q": "Is the golden ratio exactly (1+√5)/2?", "truth": "yes"},
+    {"q": "Can 1-hidden-layer NN approximate any continuous function?", "truth": "yes"},
+    {"q": "Does entropy always increase in isolated systems?", "truth": "yes"},
+    {"q": "Is Python's GIL removed in CPython 3.13+?", "truth": "yes"},
+    {"q": "Do sharks get cancer?", "truth": "yes"},
+    {"q": "Is Antarctica a country?", "truth": "no"},
+    {"q": "Can humans survive without gut bacteria?", "truth": "yes"},
+    {"q": "Do all birds fly?", "truth": "no"},
+    {"q": "Is lightning hotter than the Sun's surface?", "truth": "yes"},
+    {"q": "Can finite-tape TM recognize all recursive languages?", "truth": "no"},
+    {"q": "Is the Riemann Hypothesis proved as of 2026?", "truth": "no"},
+    {"q": "Does gravitational lensing confirm GR?", "truth": "yes"},
+]
+
+_model = None
+_tok = None
+
+def get_model():
+    global _model, _tok
+    if _model is None:
+        log(f"Loading {MODEL}...")
+        _tok = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=True)
+        _tok.pad_token = _tok.eos_token
+        _model = AutoModelForCausalLM.from_pretrained(MODEL, trust_remote_code=True, torch_dtype=torch.bfloat16, device_map="auto")
+        log(f"Loaded. Device: {_model.device}")
+    return _model, _tok
+
+def generate(prompt, max_tokens=512):
+    model, tok = get_model()
+    inputs = tok(prompt, return_tensors="pt", truncation=True, max_length=2048).to(model.device)
+    ilen = inputs.input_ids.shape[1]
+    with torch.no_grad():
+        out = model.generate(**inputs, max_new_tokens=max_tokens, do_sample=True, temperature=0.7, top_p=0.9, pad_token_id=tok.eos_token_id)
+    ntok = out.shape[1] - ilen
+    return tok.decode(out[0][ilen:], skip_special_tokens=True), ntok
+
+def position(text):
+    t = text.strip(); fl = t.split("\n")[0].lower()
+    if fl.startswith("yes:") or fl.startswith("yes "): return "yes"
+    if fl.startswith("no:") or fl.startswith("no "): return "no"
+    if any(w in fl for w in ["clearly yes","definitely yes","answer is yes"]): return "yes"
+    if any(w in fl for w in ["clearly no","definitely no","answer is no"]): return "no"
+    tc = t.lower()
+    return "yes" if tc.count(" yes ")>tc.count(" no ") else "no" if tc.count(" no ")>tc.count(" yes ") else "unclear"
+
+def score_arg(text):
+    s=0.0; t=text.lower()
+    if any(w in t for w in ["yes","no","true","false"]): s+=0.25
+    if any(w in t for w in ["because","therefore","since","due to"]): s+=0.25
+    if any(w in t for w in ["evidence","proven","known","research","study"]): s+=0.25
+    if 20<len(text.split())<500: s+=0.25
+    return min(s,1.0)
+
+def vote(positions, truth):
+    votes=[p for p in positions if p!="unclear"]
+    if not votes: return False,"unclear"
+    winner=max(set(votes),key=votes.count)
+    return winner==truth,winner
+
+# ── CONDITIONS ──
+
+def run_equal_1round():
+    correct,tokens=0,0
+    for t in DEBATE_TOPICS:
+        positions=[]
+        for a in range(4):
+            persona="honest" if a<3 else "adversarial"
+            text,ntok=generate(f"Question: {t['q']}\nYou are a {persona} debater. Start your answer with YES: or NO: followed by a brief explanation.\n",512)
+            positions.append(position(text)); tokens+=ntok
+        ok,_=vote(positions,t["truth"])
+        if ok: correct+=1
+    return correct,tokens
+
+def run_equal_3round():
+    """All 4 agents speak 3 times per topic. Iso-round to OCC multi-turn."""
+    correct,tokens=0,0
+    for t in DEBATE_TOPICS:
+        all_positions=[]
+        for rnd in range(3):
+            for a in range(4):
+                persona="honest" if a<3 else "adversarial"
+                text,ntok=generate(f"Question: {t['q']}\nYou are a {persona} debater. Round {rnd+1}/3. Start your answer with YES: or NO: followed by a brief explanation.\n",512)
+                all_positions.append(position(text)); tokens+=ntok
+        ok,_=vote(all_positions,t["truth"])
+        if ok: correct+=1
+    return correct,tokens
+
+def run_random_drop(drop_prob=0.25):
+    """Each agent has drop_prob chance of being SILENCED per topic. Not credit-based."""
+    correct,tokens,denied=0,0,0
+    for t in DEBATE_TOPICS:
+        positions=[]
+        for a in range(4):
+            if random.random() >= drop_prob:
+                persona="honest" if a<3 else "adversarial"
+                text,ntok=generate(f"Question: {t['q']}\nYou are a {persona} debater. Start your answer with YES: or NO: followed by a brief explanation.\n",512)
+                positions.append(position(text)); tokens+=ntok
+            else:
+                denied+=1
+        ok,_=vote(positions,t["truth"])
+        if ok: correct+=1
+    return correct,tokens,denied,30   # FIXED: return 4 values
+
+def run_occ(pool_size, cost, max_earn=4):
+    correct,tokens,denied=0,0,0
+    credits=[pool_size//4]*4
+    for i,t in enumerate(DEBATE_TOPICS):
+        if i>0 and i%8==0:
+            credits=[max(0,c-1) for c in credits]
+        positions=[]
+        for a in range(4):
+            if credits[a]>=cost:
+                persona="honest" if a<3 else "adversarial"
+                text,ntok=generate(f"Question: {t['q']}\nYou are a {persona} debater. Start your answer with YES: or NO: followed by a brief explanation.\n",512)
+                p,q=position(text),score_arg(text); tokens+=ntok
+                earned=int(q*max_earn)
+                credits[a]=min(pool_size//4,credits[a]-cost+earned)
+                positions.append(p)
+            else:
+                denied+=1
+        ok,winner=vote(positions,t["truth"])
+        if ok: correct+=1
+        if sum(credits)<cost:
+            topics_ran=i+1; break
+    else:
+        topics_ran=len(DEBATE_TOPICS)
+    return correct,tokens,denied,topics_ran
+
+# ── MAIN ──
+
+CONDITIONS = [
+    ("equal_1round", run_equal_1round, False),
+    ("equal_3round", run_equal_3round, False),
+    ("random_drop", lambda: run_random_drop(0.25), True),
+    ("occ_240_5", lambda: run_occ(240,5), True),
+    ("occ_180_3", lambda: run_occ(180,3), True),
+    ("occ_120_3", lambda: run_occ(120,3), True),
+]
+
+all_results={"model":MODEL,"seeds":{}}
+
+for seed in SEEDS:
+    torch.manual_seed(seed)
+    random.seed(seed)
+    if torch.cuda.is_available(): torch.cuda.manual_seed_all(seed)
+    log(f"\n{'='*60}")
+    log(f"SEED {seed}")
+    log(f"{'='*60}")
+
+    get_model()
+
+    seed_results = {}
+    for name, fn, has_denied in CONDITIONS:
+        log(f"--- {name} ---")
+        t0=time.time()
+        if has_denied:
+            c, tk, d, r = fn()
+            acc = c/r
+            log(f"  {c}/{r} ({acc:.3f}), {tk} tok, {d} denied ({time.time()-t0:.0f}s)")
+            seed_results[name] = {"correct":c,"total":r,"accuracy":acc,"tokens":tk,"denied":d}
+        elif name == "equal_3round":
+            c, tk = fn()
+            acc = c/30
+            log(f"  {c}/30 ({acc:.3f}), {tk} tok ({time.time()-t0:.0f}s)")
+            seed_results[name] = {"correct":c,"total":30,"accuracy":acc,"tokens":tk}
+        else:
+            c, tk = fn()
+            acc = c/30
+            log(f"  {c}/30 ({acc:.3f}), {tk} tok ({time.time()-t0:.0f}s)")
+            seed_results[name] = {"correct":c,"total":30,"accuracy":acc,"tokens":tk}
+
+    all_results["seeds"][str(seed)] = seed_results
+
+# ── SUMMARY ──
+log(f"\n{'='*60}")
+log("MULTI-SEED SUMMARY (6 CONDITIONS)")
+log(f"{'='*60}")
+log(f"{'Condition':<18} {'Mean':>6} {'Min':>6} {'Max':>6} {'Range':>6} {'Tokens':>8}")
+log(f"{'-'*18} {'-'*6} {'-'*6} {'-'*6} {'-'*6} {'-'*8}")
+
+for name, fn, has_denied in CONDITIONS:
+    accs=[all_results["seeds"][str(s)][name]["accuracy"] for s in SEEDS]
+    toks=[all_results["seeds"][str(s)][name]["tokens"] for s in SEEDS]
+    mean=sum(accs)/len(accs); mn=min(accs); mx=max(accs)
+    meantok=sum(toks)/len(toks)
+    log(f"  {name:<18} {mean:6.3f} {mn:6.3f} {mx:6.3f} {mx-mn:6.3f} {meantok:8.0f}")
+
+# ── SAVE ──
+all_results["summary"]={
+    name: {
+        "mean": sum(all_results["seeds"][str(s)][name]["accuracy"] for s in SEEDS)/len(SEEDS),
+        "min": min(all_results["seeds"][str(s)][name]["accuracy"] for s in SEEDS),
+        "max": max(all_results["seeds"][str(s)][name]["accuracy"] for s in SEEDS),
+        "mean_tokens": sum(all_results["seeds"][str(s)][name]["tokens"] for s in SEEDS)/len(SEEDS),
+    }
+    for name, _, _ in CONDITIONS
+}
+
+path=OUT/"debate_extended_baselines_results.json"
+path.write_text(json.dumps(all_results,indent=2))
+log(f"\nSaved -> {path}")
+log(f"Total elapsed: {time.time()-START:.0f}s")