Upload benchmarks/benchmark_debate.py

benchmarks/benchmark_debate.py

@@ -0,0 +1,419 @@
+"""
+Benchmark 3: Multi-Agent Debate Under Shared Compute
+
+Compares:
+A. equal turns
+B. majority vote
+C. confidence-weighted vote
+D. verifier-only allocation
+E. OCC credit allocation
+F. OCC with decay and non-transferability
+
+Uses simulated factual disputes and code debates.
+"""
+
+import json
+import random
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import numpy as np
+
+import sys
+sys.path.insert(0, str(Path(__file__).parent.parent))
+from oracle.oracle import ImpactOracle, OracleResult
+from ledger.ledger import CreditLedger
+from broker.broker import ResourceBroker, Decision
+
+
+@dataclass
+class DebateTopic:
+    question: str
+    correct_answer: str
+    distractors: List[str]
+
+
+class SimulatedDebateAgent:
+    """
+    Simulates a debate participant with variable accuracy and confidence.
+    """
+
+    def __init__(
+        self,
+        agent_id: str,
+        accuracy: float = 0.6,
+        confidence_bias: float = 0.1,
+        verbose_prob: float = 0.0,
+        collude_with: Optional[str] = None,
+    ):
+        self.agent_id = agent_id
+        self.accuracy = accuracy
+        self.confidence_bias = confidence_bias
+        self.verbose_prob = verbose_prob
+        self.collude_with = collude_with
+        self.tokens_used = 0
+        self.turns_taken = 0
+        self.influence_score = 0.0
+
+    def propose(self, topic: DebateTopic, prior_proposals: List[Dict]) -> Dict:
+        """Propose an answer with confidence."""
+        self.turns_taken += 1
+        tokens = 50 + random.randint(0, 50)
+        if random.random() < self.verbose_prob:
+            tokens *= 4  # verbose padding
+        self.tokens_used += tokens
+
+        # Accuracy
+        correct = random.random() < self.accuracy
+        if correct:
+            answer = topic.correct_answer
+        else:
+            answer = random.choice(topic.distractors)
+
+        # Confidence calibration
+        if correct:
+            confidence = 0.7 + random.random() * 0.3 + self.confidence_bias
+        else:
+            confidence = 0.4 + random.random() * 0.4 + self.confidence_bias
+        confidence = max(0.0, min(1.0, confidence))
+
+        # Influence: if we agree with majority, our influence is lower
+        if prior_proposals:
+            majority = max(set(p["answer"] for p in prior_proposals), key=lambda x: sum(1 for p in prior_proposals if p["answer"] == x))
+            if answer == majority:
+                influence = 0.1
+            else:
+                influence = 0.5
+        else:
+            influence = 0.3
+
+        self.influence_score += influence
+
+        return {
+            "agent_id": self.agent_id,
+            "answer": answer,
+            "confidence": confidence,
+            "correct": correct,
+            "tokens": tokens,
+            "influence": influence,
+        }
+
+
+class DebateBenchmark:
+    """
+    Benchmark multi-agent debate under shared compute budgets.
+    """
+
+    def __init__(
+        self,
+        n_topics: int = 50,
+        n_agents: int = 4,
+        budget_per_topic: float = 500.0,
+        seed: int = 42,
+    ):
+        self.n_topics = n_topics
+        self.n_agents = n_agents
+        self.budget_per_topic = budget_per_topic
+        self.seed = seed
+        self.topics: List[DebateTopic] = []
+        self.oracle = ImpactOracle(compute_budget=budget_per_topic)
+
+    def generate_topics(self):
+        random.seed(self.seed)
+        np.random.seed(self.seed)
+
+        topic_pool = [
+            ("What is 15 * 17?", "255", ["245", "265", "225", "275"]),
+            ("Capital of Australia?", "Canberra", ["Sydney", "Melbourne", "Perth", "Brisbane"]),
+            ("Author of '1984'?", "George Orwell", ["Aldous Huxley", "Ray Bradbury", "H.G. Wells", "Kurt Vonnegut"]),
+            ("Square root of 256?", "16", ["14", "18", "12", "20"]),
+            ("Element with symbol Au?", "Gold", ["Silver", "Aluminum", "Argon", "Astatine"]),
+            ("Year WWI ended?", "1918", ["1919", "1917", "1920", "1916"]),
+            ("Smallest prime number?", "2", ["1", "3", "0", "-1"]),
+            ("Largest planet?", "Jupiter", ["Saturn", "Neptune", "Uranus", "Earth"]),
+            ("Speed of light (m/s)?", "299792458", ["300000000", "299000000", "310000000", "280000000"]),
+            ("First US president?", "George Washington", ["Thomas Jefferson", "John Adams", "Abraham Lincoln", "Benjamin Franklin"]),
+        ]
+
+        for i in range(self.n_topics):
+            t = topic_pool[i % len(topic_pool)]
+            self.topics.append(DebateTopic(question=t[0], correct_answer=t[1], distractors=t[2]))
+
+    def _resolve_equal_turns(self, agents: List[SimulatedDebateAgent], topic: DebateTopic, turns_per_agent: int = 2) -> Dict:
+        """Strategy A: equal turns, then majority vote."""
+        proposals = []
+        compute_used = 0.0
+        for agent in agents:
+            for _ in range(turns_per_agent):
+                prop = agent.propose(topic, proposals)
+                proposals.append(prop)
+                compute_used += prop["tokens"]
+
+        # Majority vote (all proposals equal weight)
+        answers = [p["answer"] for p in proposals]
+        final_answer = max(set(answers), key=answers.count)
+        correct = final_answer == topic.correct_answer
+
+        return {
+            "strategy": "equal_turns",
+            "correct": correct,
+            "final_answer": final_answer,
+            "compute_used": compute_used,
+            "n_turns": len(proposals),
+            "proposals": proposals,
+        }
+
+    def _resolve_majority_vote(self, agents: List[SimulatedDebateAgent], topic: DebateTopic, turns_per_agent: int = 2) -> Dict:
+        """Strategy B: majority vote on first proposal per agent."""
+        proposals = []
+        compute_used = 0.0
+        for agent in agents:
+            prop = agent.propose(topic, proposals)
+            proposals.append(prop)
+            compute_used += prop["tokens"]
+
+        answers = [p["answer"] for p in proposals]
+        final_answer = max(set(answers), key=answers.count)
+        correct = final_answer == topic.correct_answer
+
+        return {
+            "strategy": "majority_vote",
+            "correct": correct,
+            "final_answer": final_answer,
+            "compute_used": compute_used,
+            "n_turns": len(proposals),
+            "proposals": proposals,
+        }
+
+    def _resolve_confidence_weighted(self, agents: List[SimulatedDebateAgent], topic: DebateTopic, turns_per_agent: int = 2) -> Dict:
+        """Strategy C: confidence-weighted vote."""
+        proposals = []
+        compute_used = 0.0
+        for agent in agents:
+            prop = agent.propose(topic, proposals)
+            proposals.append(prop)
+            compute_used += prop["tokens"]
+
+        # Weighted vote by confidence
+        vote_scores: Dict[str, float] = {}
+        for p in proposals:
+            vote_scores[p["answer"]] = vote_scores.get(p["answer"], 0.0) + p["confidence"]
+        final_answer = max(vote_scores, key=vote_scores.get)
+        correct = final_answer == topic.correct_answer
+
+        return {
+            "strategy": "confidence_weighted",
+            "correct": correct,
+            "final_answer": final_answer,
+            "compute_used": compute_used,
+            "n_turns": len(proposals),
+            "proposals": proposals,
+        }
+
+    def _resolve_occ_allocation(
+        self,
+        agents: List[SimulatedDebateAgent],
+        topic: DebateTopic,
+        max_turns: int = 12,
+        use_decay: bool = True,
+    ) -> Dict:
+        """Strategy E/F: OCC allocates turns based on marginal contribution."""
+        ledger = CreditLedger(decay_lambda=0.1 if use_decay else 0.0)
+        broker = ResourceBroker()
+        proposals = []
+        compute_used = 0.0
+        turns = 0
+
+        # One initial proposal from each agent
+        for agent in agents:
+            prop = agent.propose(topic, proposals)
+            proposals.append(prop)
+            compute_used += prop["tokens"]
+            turns += 1
+
+            # Score the proposal
+            oracle_res = self.oracle.score(
+                mode="debate",
+                action={"tokens_used": prop["tokens"]},
+                context={"previous_correct": False},
+                result={
+                    "final_correct": prop["correct"],
+                    "agent_contribution": prop["influence"],
+                    "compute_cost": prop["tokens"],
+                    "tokens_used": prop["tokens"],
+                    "total_turns": turns,
+                },
+                agent_id=agent.agent_id,
+            )
+
+            if prop["correct"]:
+                ledger.earn(
+                    agent_id=agent.agent_id,
+                    task_id=topic.question[:30],
+                    action_id=f"turn_{turns}",
+                    amount=oracle_res.reward_value * 5.0,
+                    oracle_score=oracle_res.raw_score,
+                    compute_cost=prop["tokens"],
+                    reason="correct_proposal",
+                )
+
+        # Iteratively allocate additional turns to best performers
+        while turns < max_turns and compute_used < self.budget_per_topic:
+            # Sort agents by credit balance
+            balances = [(a, ledger.balance(a.agent_id, "general", "global")) for a in agents]
+            balances.sort(key=lambda x: x[1], reverse=True)
+
+            # Try to give a turn to the top agent
+            top_agent, top_balance = balances[0]
+            dec = broker.request(
+                "debate_turn",
+                top_agent.agent_id,
+                top_balance,
+                task_state={"progress": sum(1 for p in proposals if p["correct"]) / len(proposals)},
+            )
+
+            if dec.decision == Decision.DENY:
+                # Try next agent
+                if len(balances) > 1:
+                    top_agent, top_balance = balances[1]
+                    dec = broker.request("debate_turn", top_agent.agent_id, top_balance, task_state={})
+                    if dec.decision == Decision.DENY:
+                        break
+                else:
+                    break
+
+            prop = top_agent.propose(topic, proposals)
+            proposals.append(prop)
+            compute_used += prop["tokens"]
+            turns += 1
+
+            # Update ledger
+            oracle_res = self.oracle.score(
+                mode="debate",
+                action={"tokens_used": prop["tokens"]},
+                context={"previous_correct": any(p["correct"] for p in proposals[:-1])},
+                result={
+                    "final_correct": prop["correct"],
+                    "agent_contribution": prop["influence"],
+                    "compute_cost": prop["tokens"],
+                    "tokens_used": prop["tokens"],
+                    "total_turns": turns,
+                },
+                agent_id=top_agent.agent_id,
+            )
+
+            if prop["correct"]:
+                ledger.earn(
+                    agent_id=top_agent.agent_id,
+                    task_id=topic.question[:30],
+                    action_id=f"turn_{turns}",
+                    amount=oracle_res.reward_value * 3.0,
+                    oracle_score=oracle_res.raw_score,
+                    compute_cost=prop["tokens"],
+                    reason="correct_proposal",
+                )
+            else:
+                # Small spend for wrong turn
+                ledger.spend(
+                    agent_id=top_agent.agent_id,
+                    task_id=topic.question[:30],
+                    action_id=f"turn_{turns}",
+                    amount=0.3,
+                    reason="wrong_proposal_cost",
+                )
+
+        # Weighted vote using final credit balances as weights
+        vote_scores: Dict[str, float] = {}
+        for p in proposals:
+            weight = ledger.balance(p["agent_id"], "general", "global")
+            weight = max(0.1, weight)
+            vote_scores[p["answer"]] = vote_scores.get(p["answer"], 0.0) + weight
+        final_answer = max(vote_scores, key=vote_scores.get)
+        correct = final_answer == topic.correct_answer
+
+        return {
+            "strategy": "occ_allocation",
+            "correct": correct,
+            "final_answer": final_answer,
+            "compute_used": compute_used,
+            "n_turns": turns,
+            "proposals": proposals,
+        }
+
+    def _summarize(self, results: List[Dict], label: str) -> Dict:
+        n = len(results)
+        correct = sum(1 for r in results if r["correct"])
+        total_compute = sum(r["compute_used"] for r in results)
+        total_turns = sum(r["n_turns"] for r in results)
+
+        return {
+            "label": label,
+            "n_topics": n,
+            "accuracy": correct / n if n else 0.0,
+            "total_compute": float(total_compute),
+            "mean_compute_per_topic": float(total_compute / n) if n else 0.0,
+            "total_turns": total_turns,
+            "mean_turns_per_topic": float(total_turns / n) if n else 0.0,
+            "quality_per_compute": (correct / n) / (total_compute / n) if total_compute else 0.0,
+            "results": results,
+        }
+
+    def run_all(self) -> Dict[str, Dict]:
+        if not self.topics:
+            self.generate_topics()
+
+        # Create agents with varied abilities
+        agents = [
+            SimulatedDebateAgent("agent_1", accuracy=0.75, confidence_bias=0.05),
+            SimulatedDebateAgent("agent_2", accuracy=0.60, confidence_bias=0.15),
+            SimulatedDebateAgent("agent_3", accuracy=0.55, confidence_bias=-0.05),
+            SimulatedDebateAgent("agent_4", accuracy=0.50, confidence_bias=0.20),
+        ]
+
+        strategies = [
+            ("equal_turns", lambda topic: self._resolve_equal_turns(agents, topic)),
+            ("majority_vote", lambda topic: self._resolve_majority_vote(agents, topic)),
+            ("confidence_weighted", lambda topic: self._resolve_confidence_weighted(agents, topic)),
+            ("occ_allocation", lambda topic: self._resolve_occ_allocation(agents, topic)),
+        ]
+
+        results = {}
+        for name, fn in strategies:
+            # Reset agents between strategies
+            for a in agents:
+                a.tokens_used = 0
+                a.turns_taken = 0
+                a.influence_score = 0.0
+
+            topic_results = []
+            for topic in self.topics:
+                topic_results.append(fn(topic))
+            results[name] = self._summarize(topic_results, name)
+
+        return results
+
+
+def main():
+    bench = DebateBenchmark(n_topics=50, n_agents=4, seed=42)
+    bench.generate_topics()
+    results = bench.run_all()
+
+    print("=" * 60)
+    print("MULTI-AGENT DEBATE BENCHMARK")
+    print("=" * 60)
+    for label, res in results.items():
+        print(f"\n{label}")
+        print(f"  accuracy: {res['accuracy']:.3f}")
+        print(f"  mean compute/topic: {res['mean_compute_per_topic']:.1f}")
+        print(f"  mean turns/topic: {res['mean_turns_per_topic']:.1f}")
+        print(f"  quality per compute: {res['quality_per_compute']:.6f}")
+
+    Path("/app/occ/reports").mkdir(parents=True, exist_ok=True)
+    with open("/app/occ/reports/benchmark_debate_results.json", "w") as f:
+        json.dump(results, f, indent=2, default=str)
+    print("\nSaved to reports/benchmark_debate_results.json")
+
+
+if __name__ == "__main__":
+    main()