chimera/benchmark.py

#!/usr/bin/env python3
"""Chimera Engine Benchmark — FSI_FELON v4.0"""
import sys, os, json, time, random
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from chimera.engine import ChimeraEngine
from chimera.organs import OrganType, ORGAN_DESCRIPTIONS

TASKS = [
    "build a REST API with user authentication",
    "remember the database schema for the user module",
    "analyze this code for SQL injection vulnerabilities",
    "plan the architecture for a distributed microservice system",
    "imagine a world where code writes itself through pheromone trails",
    "search for primary sources on the origin of the Creeper virus",
    "generate a Python decorator that logs function calls",
    "store the encryption key configuration for future reference",
    "what if the nanobot swarm could communicate through dreams",
    "check this payment gateway integration for security flaws",
    "design a migration strategy from monolith to microservices",
    "recall the JWT authentication implementation from last session",
    "build a command-line tool for file encryption",
    "simulate a network of 1000 nanobots processing tasks in parallel",
    "audit the login system for race conditions",
    "save the API endpoint configuration for the notification service",
    "plan the implementation of a real-time collaborative editor",
    "imagine an intelligence that emerges from decaying pheromone trails",
    "search for evidence about the effectiveness of behavioral biometrics",
    "generate unit tests for the payment processing module",
]

ROUTING_EXPECTATIONS = {
    "build": ["prefrontal", "cerebellum", "spine"],
    "remember": ["hippocampus"],
    "analyze": ["spine", "prefrontal"],
    "plan": ["prefrontal"],
    "imagine": ["dream"],
    "what if": ["dream", "prefrontal"],
    "search": ["antennae", "spine"],
    "generate": ["spine", "prefrontal"],
    "store": ["hippocampus"],
    "check": ["spine", "prefrontal"],
    "design": ["prefrontal"],
    "recall": ["hippocampus"],
    "build": ["spine", "cerebellum"],
    "simulate": ["dream", "prefrontal"],
    "audit": ["spine", "prefrontal"],
    "save": ["hippocampus"],
}

def determine_expected(input_text: str) -> list:
    text_lower = input_text.lower()
    for key, expected in ROUTING_EXPECTATIONS.items():
        if text_lower.startswith(key):
            return expected
    return ["antennae"]

def run_benchmark():
    print("╔══════════════════════════════════════════════════╗")
    print("║     CHIMERA ENGINE — BENCHMARK v1.0             ║")
    print("╚══════════════════════════════════════════════════╝")

    print(f"\n  Organs ({len(OrganType)}):")
    for ot in OrganType:
        print(f"    • {ot.value:15s} — {ORGAN_DESCRIPTIONS[ot]}")

    engine = ChimeraEngine(decay_rate=0.05)
    print(f"\n  Routing {len(TASKS)} tasks through Chimera Engine...")
    print()

    results = []
    correct_routes = 0
    organ_utilization = {ot.value: 0 for ot in OrganType}

    for i, task_input in enumerate(TASKS):
        t0 = time.time()
        outcome = engine.route(task_input)
        elapsed = time.time() - t0

        routed_organ = outcome.get("routed_to", "unknown")
        expected_organs = determine_expected(task_input)
        is_correct = routed_organ in expected_organs

        if is_correct:
            correct_routes += 1
        if routed_organ in organ_utilization:
            organ_utilization[routed_organ] += 1

        mark = "✅" if is_correct else "❌"
        print(f"  {mark} [{i+1:2d}/{len(TASKS)}] {routed_organ:15s} ← {task_input[:45]:45s}  ({elapsed:.3f}s)")
        results.append({
            "task": task_input, "routed_to": routed_organ,
            "expected": expected_organs, "correct": is_correct,
            "time": round(elapsed, 4), "scores": outcome.get("all_scores", {}),
        })

    routing_accuracy = correct_routes / len(TASKS)
    print(f"\n{'='*60}")
    print(f"  RESULTS")
    print(f"{'='*60}")
    print(f"  Total tasks:          {len(TASKS)}")
    print(f"  Correct routes:       {correct_routes}")
    print(f"  Routing accuracy:     {routing_accuracy:.1%}")
    print(f"  Organs functional:    {sum(1 for s in engine.organ_stats().values() if s['tasks_processed'] > 0)}/{len(OrganType)}")

    print(f"\n  ── Organ Utilization ──")
    for ot in OrganType:
        count = organ_utilization[ot.value]
        pct = count / len(TASKS) * 100
        bar = "█" * int(pct / 2) + "░" * (50 - int(pct / 2))
        print(f"  {ot.value:15s} │{bar}│ {count:3d} tasks ({pct:.0f}%)")

    print(f"\n  ── Organ Stats ──")
    for name, stats in engine.organ_stats().items():
        sr = stats.get("success_rate", 0)
        print(f"  {name:15s} processed={stats['tasks_processed']} success_rate={sr:.0%} energy={stats['energy']}")

    print(f"\n  ── Pheromone Stats ──")
    ps = engine.pheromones.stats()
    print(f"  Active trails:        {ps['active_trails']}")
    print(f"  Total lays:           {ps['total_lays']}")
    print(f"  Total reinforcements: {ps['total_reinforcements']}")
    print(f"  Decay rate:           {ps['decay_rate']}")

    verdict = "✅ PASS" if routing_accuracy >= 0.85 and sum(1 for s in engine.organ_stats().values() if s['tasks_processed'] > 0) >= 6 else "❌ NEEDS WORK"
    print(f"\n  VERDICT: {verdict}")
    print(f"  Target:  >85% routing accuracy, 6/6 organs active")
    print(f"  Actual:  {routing_accuracy:.1%} accuracy, {sum(1 for s in engine.organ_stats().values() if s['tasks_processed'] > 0)}/6 organs active")

    report = {
        "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"),
        "benchmark": "Chimera Engine v1.0",
        "total_tasks": len(TASKS),
        "correct_routes": correct_routes,
        "routing_accuracy": round(routing_accuracy, 3),
        "organs_functional": sum(1 for s in engine.organ_stats().values() if s['tasks_processed'] > 0),
        "organ_utilization": organ_utilization,
        "organ_stats": engine.organ_stats(),
        "pheromone_stats": ps,
        "results": results,
        "verdict": "PASS" if routing_accuracy >= 0.85 else "NEEDS_WORK",
    }
    with open("/tmp/fsi_felon/chimera/benchmark_results.json", "w") as f:
        json.dump(report, f, indent=2)
    print(f"\n  Report saved: /tmp/fsi_felon/chimera/benchmark_results.json")
    return routing_accuracy

if __name__ == "__main__":
    run_benchmark()