app.py

import asyncio
import base64
from contextlib import asynccontextmanager
from datetime import datetime, timezone
from enum import Enum
import json
import logging
import os
import random
import re
import shutil
import sqlite3
import sys
import time
from typing import Any, Dict, List, Optional, Set
import uuid

from fastapi import BackgroundTasks, FastAPI, HTTPException, Query, WebSocket, WebSocketDisconnect, status
from fastapi.responses import HTMLResponse
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel, Field
import uvicorn

# Try importing psutil for real hardware monitoring; fall back gracefully if not installed.
try:
    import psutil
except ImportError:
    psutil = None


# --- LOGGING SETUP ---
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(name)s - %(message)s",
)
logger = logging.getLogger("SparkColonyOS")


# --- ENUMS & DOMAIN MODELS ---
class AppState(str, Enum):
    BOOTING = "BOOTING"
    READY = "READY"
    BUSY = "BUSY"
    PAUSED = "PAUSED"
    ERROR = "ERROR"
    RECOVERY = "RECOVERY"
    SHUTDOWN = "SHUTDOWN"


class AgentState(str, Enum):
    IDLE = "Idle"
    ASSIGNED = "Assigned"
    PLANNING = "Planning"
    SEARCHING = "Searching"
    BROWSING = "Browsing"
    READING = "Reading"
    REASONING = "Reasoning"
    DISCUSSING = "Discussing"
    WRITING = "Writing"
    REVIEWING = "Reviewing"
    COMPLETED = "Completed"
    FAILED = "Failed"
    RETRYING = "Retrying"
    SLEEPING = "Sleeping"


class MissionStatus(str, Enum):
    INITIALIZING = "Initializing"
    IN_PROGRESS = "In Progress"
    PAUSED = "Paused"
    CANCELLED = "Cancelled"
    COMPLETED = "Completed"
    FAILED = "Failed"


class DecisionStage(str, Enum):
    RECEIVE_MISSION = "Receive Mission"
    UNDERSTAND_GOAL = "Understand Goal"
    BREAK_SUBTASKS = "Break Into Subtasks"
    ESTIMATE_COST = "Estimate Cost"
    SELECT_AGENTS = "Select Agents"
    ASSIGN_TASKS = "Assign Tasks"
    EXECUTE = "Execute"
    REVIEW = "Review"
    IMPROVE = "Improve"
    FINALIZE = "Finalize"
    ARCHIVE = "Archive"


class ThinkingMode(str, Enum):
    FAST = "Fast"
    RESEARCH = "Research"
    ANALYTICAL = "Analytical"
    CREATIVE = "Creative"
    CRITICAL = "Critical"
    HISTORICAL = "Historical"
    SCIENTIFIC = "Scientific"


class ReasoningCycleStep(str, Enum):
    OBSERVE = "Observe"
    UNDERSTAND = "Understand"
    RECALL_MEMORY = "Recall Memory"
    SEARCH_KNOWLEDGE = "Search Knowledge"
    REASON = "Reason"
    GENERATE_PLAN = "Generate Plan"
    ESTIMATE_RISK = "Estimate Risk"
    EXECUTE = "Execute"
    EVALUATE = "Evaluate"
    REFLECT = "Reflect"
    LEARN = "Learn"


class SourceTier(str, Enum):
    TIER_1_OFFICIAL = "Tier 1: Official/Academic"
    TIER_2_NEWS = "Tier 2: Reputable News"
    TIER_3_COMMUNITY = "Tier 3: Community Knowledge"
    TIER_4_FORUMS = "Tier 4: Forums & Discussions"
    TIER_5_UNTRUSTED = "Tier 5: Social Media / Untrusted"


class PopupType(str, Enum):
    COOKIE_BANNER = "Cookie Banner"
    NEWSLETTER = "Newsletter Overlay"
    ADVERTISEMENT = "Advertisement"
    CHAT_WIDGET = "Chat Widget"
    CAPTCHA = "CAPTCHA Security Challenge"
    AGE_VERIFICATION = "Age Verification"
    NONE = "None"


class SafetyLevel(str, Enum):
    READ_ONLY = "Read-Only (Safe)"
    INTERACTIVE_FORM = "Interactive Form"
    SENSITIVE_ACTION = "Sensitive Action (Requires Approval)"
    DESTRUCTIVE_BLOCKED = "Destructive (Blocked)"


class DecisionScore(BaseModel):
    benefit: float = Field(..., ge=0, le=100)
    cost: float = Field(..., ge=0, le=100)
    risk: float = Field(..., ge=0, le=100)
    confidence: float = Field(..., ge=0, le=100)
    resource_usage: float = Field(..., ge=0, le=100)
    expected_value: float


class AgentMessage(BaseModel):
    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
    sender: str
    recipient: str
    mission_id: str
    status: str
    summary: str
    next_request: str
    confidence: float
    timestamp: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat())


class EvidenceScore(BaseModel):
    credibility: float = Field(..., ge=0, le=100)
    freshness: float = Field(..., ge=0, le=100)
    authority: float = Field(..., ge=0, le=100)
    agreement: float = Field(..., ge=0, le=100)
    conflict: float = Field(..., ge=0, le=100)
    overall_confidence: float = Field(..., ge=0, le=100)


class PageElement(BaseModel):
    element_type: str
    label: str
    selector: str
    bounding_box: Dict[str, float]
    confidence: float


class VisionOutput(BaseModel):
    page_summary: str
    detected_buttons: List[PageElement] = Field(default_factory=list)
    detected_inputs: List[PageElement] = Field(default_factory=list)
    detected_links: List[PageElement] = Field(default_factory=list)
    popup_detected: PopupType = PopupType.NONE
    captcha_present: bool = False
    visual_hierarchy: Dict[str, Any] = Field(default_factory=dict)
    accessibility_notes: List[str] = Field(default_factory=list)
    navigation_suggestions: List[str] = Field(default_factory=list)
    confidence: float = 100.0


class PageObservation(BaseModel):
    url: str
    title: str
    has_captcha: bool
    popup_type: PopupType
    main_content_excerpt: str
    elements_count: int
    observed_at: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat())


class PageDiffResult(BaseModel):
    previous_url: str
    current_url: str
    change_percentage: float
    added_elements: List[str]
    removed_elements: List[str]
    summary: str


class CreateMissionRequest(BaseModel):
    topic: str = Field(..., min_length=3, max_length=500, description="Research topic or directive for the colony.")
    max_depth: Optional[int] = Field(default=3, ge=1, le=10, description="Max depth of sub-research tasks.")
    thinking_mode: Optional[ThinkingMode] = Field(default=ThinkingMode.RESEARCH, description="Thinking mode for the mission.")


class ObservePageRequest(BaseModel):
    url: str
    mission_id: str = "manual"


class NavigatePageRequest(BaseModel):
    url: str
    target_action: str
    mission_id: str = "manual"


class AddMemoryRequest(BaseModel):
    agent_id: str
    mission_id: str
    content: str
    tags: List[str] = Field(default_factory=list)


class CachePageRequest(BaseModel):
    url: str
    html: str
    title: Optional[str] = "Untitled Page"


class UpdateConfigRequest(BaseModel):
    max_cost_per_mission: Optional[float] = None
    daily_budget: Optional[float] = None
    search_depth: Optional[int] = None
    browser_pool_size: Optional[int] = None
    timeout_seconds: Optional[int] = None
    vision_threshold: Optional[float] = None


class RegisterPluginRequest(BaseModel):
    name: str
    version: str
    description: str
    entry_point: str
    permissions: List[str] = Field(default_factory=list)


class MissionResponse(BaseModel):
    mission_id: str
    topic: str
    status: MissionStatus
    current_stage: DecisionStage
    created_at: str
    message: str


class AgentStatusModel(BaseModel):
    agent_id: str
    name: str
    role: str
    state: AgentState
    current_task: Optional[str]
    confidence: float
    last_active: str
    enabled: bool = True
    capabilities: List[str] = Field(default_factory=list)
    tools: List[str] = Field(default_factory=list)
    current_load: float = 0.0
    is_dynamic: bool = False


class MissionDetailResponse(BaseModel):
    mission_id: str
    topic: str
    status: MissionStatus
    stage: DecisionStage
    created_at: str
    updated_at: str
    summary: Optional[str]
    total_cost_usd: float
    messages: List[Dict[str, Any]]
    journals: List[Dict[str, Any]]
    memories: List[Dict[str, Any]]
    evidence: List[Dict[str, Any]]


class SystemStatusResponse(BaseModel):
    app_state: AppState
    uptime_seconds: float
    active_agents: int
    total_missions: int
    total_memories: int
    total_messages: int
    total_cost_usd: float
    cpu_usage_percent: float
    memory_usage_percent: float
    agents: List[AgentStatusModel]


class KnowledgeNode(BaseModel):
    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
    mission_id: str
    label: str
    entity_type: str
    confidence: float


class KnowledgeEdge(BaseModel):
    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
    mission_id: str
    source_node_id: str
    target_node_id: str
    relationship: str


# --- V2 DOMAIN MODELS ---
class BlackboardEntry(BaseModel):
    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
    mission_id: str
    agent_id: str
    topic: str
    data: Dict[str, Any]
    version: int = 1
    tags: List[str] = Field(default_factory=list)
    priority: int = Field(default=5, ge=1, le=10)
    confidence: float = 100.0
    timestamp: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat())


class AgentCapabilityProfile(BaseModel):
    capabilities: List[str] = Field(default_factory=list)
    tools: List[str] = Field(default_factory=list)
    current_model: str = "gpt-4o / gemini-2.5-flash"
    experience: int = 100
    current_load: float = 0.0
    confidence: float = 100.0
    mission_history: List[str] = Field(default_factory=list)
    success_rate: float = 100.0
    avg_duration_seconds: float = 1.25


class MissionContextModel(BaseModel):
    mission_id: str
    topic: str
    priority: int = Field(default=5, ge=1, le=10)
    current_phase: DecisionStage
    progress_percent: float = 0.0
    owner_agent: str
    resource_usage: Dict[str, float] = Field(default_factory=dict)
    estimated_cost: float = 0.0
    health_status: str = "HEALTHY"
    retry_count: int = 0
    checkpoint_state: Optional[str] = None
    created_at: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat())


class ResourceMetricsModel(BaseModel):
    cpu_usage_percent: float
    memory_usage_percent: float
    browser_instances: int
    open_tabs: int
    gemini_tokens_used: int
    groq_tokens_used: int
    sqlite_queries_count: int
    vector_searches_count: int
    task_queue_depth: int
    timestamp: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat())


class SpawnAgentRequest(BaseModel):
    role: str = Field(..., description="Role of the agent to spawn, e.g. Researcher, Browser, Vision, Writer")
    mission_id: str
    capabilities: List[str] = Field(default_factory=list)
    tools: List[str] = Field(default_factory=list)


class DynamicAgentInfo(BaseModel):
    agent_id: str
    name: str
    role: str
    mission_id: str
    is_dynamic: bool = True
    created_at: str


# --- V2 PHASE 2 DOMAIN MODELS ---
class LogicalModel(str, Enum):
    MDL_FST = "mdl_fst"  # Fast reasoning, UI understanding, OCR, Vision, Planning
    MDL_ADV = "mdl_adv"  # Deep reasoning, Long reports, Research, Synthesis


class KeyStatusModel(BaseModel):
    key_id: str
    provider: str
    is_busy: bool
    is_disabled: bool
    error_count: int
    total_calls: int
    cooldown_remaining_sec: float


class ModelTelemetryModel(BaseModel):
    logical_models: Dict[str, str]
    total_keys_managed: int
    active_keys_count: int
    disabled_keys_count: int
    total_model_calls: int
    key_telemetry: List[KeyStatusModel]


class ToolDefinition(BaseModel):
    name: str
    category: str
    description: str
    permission_level: SafetyLevel
    enabled: bool = True


class ToolRequest(BaseModel):
    agent_id: str
    mission_id: str
    tool_name: str
    parameters: Dict[str, Any] = Field(default_factory=dict)


class ToolResponse(BaseModel):
    tool_name: str
    allowed: bool
    reason: str
    result: Optional[Any] = None


class TimelineStepType(str, Enum):
    THINKING = "Thinking"
    SEARCHING = "Searching"
    BROWSING = "Browsing"
    READING = "Reading"
    VISION = "Vision"
    REASONING = "Reasoning"
    WRITING = "Writing"
    REFLECTION = "Reflection"
    COMPLETION = "Completion"


class TimelineEventModel(BaseModel):
    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
    mission_id: str
    agent_id: str
    step_type: TimelineStepType
    description: str
    metadata: Dict[str, Any] = Field(default_factory=dict)
    timestamp: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat())


# --- V2 PHASE 3 DOMAIN MODELS ---
class DiscussionType(str, Enum):
    AGREE = "Agree"
    DISAGREE = "Disagree"
    QUESTION = "Question"
    SUGGEST = "Suggest"
    CRITICIZE = "Criticize"
    REQUEST_CLARIFICATION = "Request Clarification"
    EVIDENCE_PROVIDE = "Provide Evidence"
    STRATEGY_PROPOSE = "Propose Strategy"


class DiscussionEntry(BaseModel):
    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
    mission_id: str
    agent_id: str
    agent_name: str
    discussion_type: DiscussionType
    topic: str
    content: str
    evidence_ref: Optional[str] = None
    tags: List[str] = Field(default_factory=list)
    confidence: float = 100.0
    timestamp: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat())


class DebateTurn(BaseModel):
    turn_number: int
    agent_id: str
    agent_name: str
    position: str
    argument: str
    evidence_claims: List[str] = Field(default_factory=list)
    confidence: float = 100.0
    timestamp: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat())


class DebateSession(BaseModel):
    debate_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
    mission_id: str
    topic: str
    status: str = "IN_PROGRESS"  # IN_PROGRESS, CONSENSUS_REACHED, CONCLUDED
    participants: List[str] = Field(default_factory=list)
    turns: List[DebateTurn] = Field(default_factory=list)
    consensus_summary: Optional[str] = None
    final_confidence: float = 0.0
    created_at: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat())


class ConsensusRequest(BaseModel):
    mission_id: str
    topic: str
    evidence_confidence: float = Field(..., ge=0, le=100)
    agreement_score: float = Field(..., ge=0, le=100)
    source_quality: float = Field(..., ge=0, le=100)
    historical_accuracy: float = Field(..., ge=0, le=100)
    memory_similarity: float = Field(..., ge=0, le=100)
    mission_risk: float = Field(..., ge=0, le=100)
    model_confidence: float = Field(..., ge=0, le=100)


class ConsensusResult(BaseModel):
    mission_id: str
    topic: str
    composite_consensus_score: float
    decision_recommendation: str
    risk_assessment: str
    confidence_tier: str


class TaskNegotiationProposal(BaseModel):
    proposal_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
    mission_id: str
    proposing_agent: str
    task_description: str
    proposed_action: str  # "ASSIGN", "SKIP_DUPLICATE", "DELEGATE"
    reasoning: str


class TaskNegotiationResult(BaseModel):
    proposal_id: str
    accepted: bool
    assigned_agent: str
    resolution_notes: str


class AgentReputationModel(BaseModel):
    agent_id: str
    name: str
    role: str
    experience_points: int = 100
    trust_score: float = 100.0
    accuracy_rate: float = 100.0
    reliability_score: float = 100.0
    speed_score: float = 100.0
    cost_efficiency: float = 100.0
    avg_confidence: float = 100.0
    success_rate: float = 100.0
    total_missions_participated: int = 0


class AgentReflectionDetail(BaseModel):
    reflection_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
    mission_id: str
    agent_id: str
    agent_name: str
    what_worked: str
    what_failed: str
    what_surprised: str
    what_to_improve: str
    timestamp: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat())


# --- V2 PHASE 4 DOMAIN MODELS ---
class ApprovalStatus(str, Enum):
    PENDING = "PENDING"
    APPROVED = "APPROVED"
    REJECTED = "REJECTED"


class ActionType(str, Enum):
    LOGIN_CONFIRMATION = "LOGIN_CONFIRMATION"
    OTP_ENTRY = "OTP_ENTRY"
    SENSITIVE_FORM = "SENSITIVE_FORM"
    FILE_UPLOAD = "FILE_UPLOAD"
    EXTERNAL_PUBLISHING = "EXTERNAL_PUBLISHING"


class ApprovalRequestModel(BaseModel):
    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
    mission_id: str
    agent_id: str
    action_type: ActionType
    prompt_message: str
    status: ApprovalStatus = ApprovalStatus.PENDING
    input_data: Dict[str, Any] = Field(default_factory=dict)
    created_at: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat())


class NotificationLevel(str, Enum):
    INFO = "INFO"
    WARNING = "WARNING"
    URGENT = "URGENT"
    ACTION_REQUIRED = "ACTION_REQUIRED"


class NotificationModel(BaseModel):
    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
    level: NotificationLevel
    title: str
    message: str
    mission_id: Optional[str] = None
    acknowledged: bool = False
    created_at: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat())


class ConversationMessageModel(BaseModel):
    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
    user_id: str = "human-operator"
    sender: str
    message: str
    metadata: Dict[str, Any] = Field(default_factory=dict)
    timestamp: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat())


class ChatRequest(BaseModel):
    message: str
    user_id: Optional[str] = "human-operator"


class ResolveApprovalRequest(BaseModel):
    approved: bool
    input_data: Optional[Dict[str, Any]] = Field(default_factory=dict)


class ModifyMissionRequest(BaseModel):
    new_topic: Optional[str] = None
    insert_subtask: Optional[str] = None
    priority: Optional[int] = None


# --- CONFIGURATION ENGINE ---
class ConfigEngine:
    """Manages runtime system settings dynamically."""

    def __init__(self):
        self.max_cost_per_mission: float = 1.00
        self.daily_budget: float = 10.00
        self.search_depth: int = 3
        self.browser_pool_size: int = 5
        self.timeout_seconds: int = 120
        self.vision_threshold: float = 0.85

    def update(self, req: UpdateConfigRequest) -> Dict[str, Any]:
        if req.max_cost_per_mission is not None:
            self.max_cost_per_mission = req.max_cost_per_mission
        if req.daily_budget is not None:
            self.daily_budget = req.daily_budget
        if req.search_depth is not None:
            self.search_depth = req.search_depth
        if req.browser_pool_size is not None:
            self.browser_pool_size = req.browser_pool_size
        if req.timeout_seconds is not None:
            self.timeout_seconds = req.timeout_seconds
        if req.vision_threshold is not None:
            self.vision_threshold = req.vision_threshold
        return self.to_dict()

    def to_dict(self) -> Dict[str, Any]:
        return {
            "max_cost_per_mission": self.max_cost_per_mission,
            "daily_budget": self.daily_budget,
            "search_depth": self.search_depth,
            "browser_pool_size": self.browser_pool_size,
            "timeout_seconds": self.timeout_seconds,
            "vision_threshold": self.vision_threshold,
        }


# --- DATABASE LAYER ---
class DatabaseManager:
    """SQLite Database Engine for Spark Colony System State, Memory & Logs."""

    def __init__(self, db_path: str = "spark_colony.db"):
        self.db_path = db_path
        self._init_db()

    def _get_connection(self) -> sqlite3.Connection:
        conn = sqlite3.connect(self.db_path, check_same_thread=False)
        conn.row_factory = sqlite3.Row
        conn.execute("PRAGMA journal_mode=WAL;")
        return conn

    def _init_db(self) -> None:
        with self._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS missions (
                    id TEXT PRIMARY KEY,
                    topic TEXT NOT NULL,
                    status TEXT NOT NULL,
                    stage TEXT NOT NULL,
                    created_at TEXT NOT NULL,
                    updated_at TEXT NOT NULL,
                    summary TEXT,
                    total_cost REAL DEFAULT 0.0
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS agents (
                    id TEXT PRIMARY KEY,
                    name TEXT NOT NULL,
                    role TEXT NOT NULL,
                    state TEXT NOT NULL,
                    current_task TEXT,
                    confidence REAL NOT NULL,
                    last_active TEXT NOT NULL,
                    enabled INTEGER DEFAULT 1
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS memories (
                    id TEXT PRIMARY KEY,
                    agent_id TEXT NOT NULL,
                    mission_id TEXT NOT NULL,
                    content TEXT NOT NULL,
                    tags TEXT,
                    created_at TEXT NOT NULL
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS journals (
                    id TEXT PRIMARY KEY,
                    agent_id TEXT NOT NULL,
                    mission_id TEXT NOT NULL,
                    entry TEXT NOT NULL,
                    timestamp TEXT NOT NULL
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS messages (
                    id TEXT PRIMARY KEY,
                    sender TEXT NOT NULL,
                    recipient TEXT NOT NULL,
                    mission_id TEXT NOT NULL,
                    status TEXT NOT NULL,
                    summary TEXT NOT NULL,
                    next_request TEXT NOT NULL,
                    confidence REAL NOT NULL,
                    timestamp TEXT NOT NULL
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS evidence (
                    id TEXT PRIMARY KEY,
                    mission_id TEXT NOT NULL,
                    agent_id TEXT NOT NULL,
                    claim TEXT NOT NULL,
                    source TEXT NOT NULL,
                    credibility REAL NOT NULL,
                    freshness REAL NOT NULL,
                    authority REAL NOT NULL,
                    agreement REAL NOT NULL,
                    conflict REAL NOT NULL,
                    confidence REAL NOT NULL,
                    created_at TEXT NOT NULL
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS token_costs (
                    id TEXT PRIMARY KEY,
                    mission_id TEXT NOT NULL,
                    agent_id TEXT NOT NULL,
                    prompt_tokens INT DEFAULT 0,
                    completion_tokens INT DEFAULT 0,
                    reasoning_tokens INT DEFAULT 0,
                    vision_tokens INT DEFAULT 0,
                    cost_usd REAL DEFAULT 0.0,
                    timestamp TEXT NOT NULL
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS browser_cache (
                    id TEXT PRIMARY KEY,
                    url TEXT UNIQUE NOT NULL,
                    html TEXT NOT NULL,
                    title TEXT,
                    cached_at TEXT NOT NULL
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS plugins (
                    id TEXT PRIMARY KEY,
                    name TEXT UNIQUE NOT NULL,
                    version TEXT NOT NULL,
                    description TEXT NOT NULL,
                    entry_point TEXT NOT NULL,
                    permissions TEXT,
                    status TEXT NOT NULL,
                    registered_at TEXT NOT NULL
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS reflections (
                    id TEXT PRIMARY KEY,
                    mission_id TEXT NOT NULL,
                    agent_id TEXT NOT NULL,
                    lessons_learned TEXT NOT NULL,
                    mistakes_identified TEXT,
                    cost_usd REAL NOT NULL,
                    confidence_achieved REAL NOT NULL,
                    created_at TEXT NOT NULL,
                    FOREIGN KEY (mission_id) REFERENCES missions (id)
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS website_profiles (
                    domain TEXT PRIMARY KEY,
                    trust_score REAL NOT NULL,
                    authority REAL NOT NULL,
                    typical_layout TEXT NOT NULL,
                    has_captcha_history INTEGER DEFAULT 0,
                    interaction_success_rate REAL DEFAULT 100.0,
                    last_visited TEXT NOT NULL
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS screen_memories (
                    id TEXT PRIMARY KEY,
                    mission_id TEXT NOT NULL,
                    url TEXT NOT NULL,
                    screenshot_ref TEXT NOT NULL,
                    layout_summary TEXT NOT NULL,
                    timestamp TEXT NOT NULL
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS downloaded_files (
                    id TEXT PRIMARY KEY,
                    mission_id TEXT NOT NULL,
                    filename TEXT NOT NULL,
                    mime_type TEXT NOT NULL,
                    file_size INTEGER NOT NULL,
                    summary TEXT,
                    created_at TEXT NOT NULL
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS checkpoints (
                    id TEXT PRIMARY KEY,
                    mission_id TEXT NOT NULL,
                    stage TEXT NOT NULL,
                    data_json TEXT NOT NULL,
                    created_at TEXT NOT NULL,
                    FOREIGN KEY (mission_id) REFERENCES missions (id)
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS knowledge_nodes (
                    id TEXT PRIMARY KEY,
                    mission_id TEXT NOT NULL,
                    label TEXT NOT NULL,
                    entity_type TEXT NOT NULL,
                    confidence REAL NOT NULL
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS knowledge_edges (
                    id TEXT PRIMARY KEY,
                    mission_id TEXT NOT NULL,
                    source_node_id TEXT NOT NULL,
                    target_node_id TEXT NOT NULL,
                    relationship TEXT NOT NULL,
                    FOREIGN KEY (source_node_id) REFERENCES knowledge_nodes (id),
                    FOREIGN KEY (target_node_id) REFERENCES knowledge_nodes (id)
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS blackboard (
                    id TEXT PRIMARY KEY,
                    mission_id TEXT NOT NULL,
                    agent_id TEXT NOT NULL,
                    topic TEXT NOT NULL,
                    data_json TEXT NOT NULL,
                    timestamp TEXT NOT NULL
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS mission_contexts (
                    mission_id TEXT PRIMARY KEY,
                    topic TEXT NOT NULL,
                    priority INTEGER DEFAULT 5,
                    current_phase TEXT NOT NULL,
                    progress_percent REAL DEFAULT 0.0,
                    owner_agent TEXT NOT NULL,
                    resource_usage_json TEXT NOT NULL,
                    estimated_cost REAL DEFAULT 0.0,
                    health_status TEXT DEFAULT 'HEALTHY',
                    retry_count INTEGER DEFAULT 0,
                    checkpoint_state TEXT,
                    created_at TEXT NOT NULL
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS mission_timeline (
                    id TEXT PRIMARY KEY,
                    mission_id TEXT NOT NULL,
                    agent_id TEXT NOT NULL,
                    step_type TEXT NOT NULL,
                    description TEXT NOT NULL,
                    metadata_json TEXT NOT NULL,
                    timestamp TEXT NOT NULL
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS discussions (
                    id TEXT PRIMARY KEY,
                    mission_id TEXT NOT NULL,
                    agent_id TEXT NOT NULL,
                    agent_name TEXT NOT NULL,
                    discussion_type TEXT NOT NULL,
                    topic TEXT NOT NULL,
                    content TEXT NOT NULL,
                    evidence_ref TEXT,
                    tags_json TEXT,
                    confidence REAL NOT NULL,
                    timestamp TEXT NOT NULL
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS debates (
                    debate_id TEXT PRIMARY KEY,
                    mission_id TEXT NOT NULL,
                    topic TEXT NOT NULL,
                    status TEXT NOT NULL,
                    participants_json TEXT NOT NULL,
                    turns_json TEXT NOT NULL,
                    consensus_summary TEXT,
                    final_confidence REAL NOT NULL,
                    created_at TEXT NOT NULL
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS agent_reputations (
                    agent_id TEXT PRIMARY KEY,
                    name TEXT NOT NULL,
                    role TEXT NOT NULL,
                    experience_points INTEGER DEFAULT 100,
                    trust_score REAL DEFAULT 100.0,
                    accuracy_rate REAL DEFAULT 100.0,
                    reliability_score REAL DEFAULT 100.0,
                    speed_score REAL DEFAULT 100.0,
                    cost_efficiency REAL DEFAULT 100.0,
                    avg_confidence REAL DEFAULT 100.0,
                    success_rate REAL DEFAULT 100.0,
                    total_missions_participated INTEGER DEFAULT 0
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS agent_reflections_v2 (
                    reflection_id TEXT PRIMARY KEY,
                    mission_id TEXT NOT NULL,
                    agent_id TEXT NOT NULL,
                    agent_name TEXT NOT NULL,
                    what_worked TEXT NOT NULL,
                    what_failed TEXT NOT NULL,
                    what_surprised TEXT NOT NULL,
                    what_to_improve TEXT NOT NULL,
                    timestamp TEXT NOT NULL
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS approval_requests (
                    id TEXT PRIMARY KEY,
                    mission_id TEXT NOT NULL,
                    agent_id TEXT NOT NULL,
                    action_type TEXT NOT NULL,
                    prompt_message TEXT NOT NULL,
                    status TEXT NOT NULL,
                    input_data_json TEXT,
                    created_at TEXT NOT NULL
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS notifications (
                    id TEXT PRIMARY KEY,
                    level TEXT NOT NULL,
                    title TEXT NOT NULL,
                    message TEXT NOT NULL,
                    mission_id TEXT,
                    acknowledged INTEGER DEFAULT 0,
                    created_at TEXT NOT NULL
                )
            """)
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS conversation_logs (
                    id TEXT PRIMARY KEY,
                    user_id TEXT NOT NULL,
                    sender TEXT NOT NULL,
                    message TEXT NOT NULL,
                    metadata_json TEXT,
                    timestamp TEXT NOT NULL
                )
            """)
            # Safe schema migration for blackboard
            try:
                cursor.execute("ALTER TABLE blackboard ADD COLUMN version INTEGER DEFAULT 1")
                cursor.execute("ALTER TABLE blackboard ADD COLUMN tags_json TEXT DEFAULT '[]'")
                cursor.execute("ALTER TABLE blackboard ADD COLUMN priority INTEGER DEFAULT 5")
                cursor.execute("ALTER TABLE blackboard ADD COLUMN confidence REAL DEFAULT 100.0")
            except sqlite3.OperationalError:
                pass  # Columns already exist
            conn.commit()

    async def save_mission(
        self,
        mission_id: str,
        topic: str,
        status: MissionStatus,
        stage: DecisionStage,
        summary: Optional[str] = None,
        total_cost: float = 0.0,
    ) -> None:
        def _exec():
            now = datetime.now(timezone.utc).isoformat()
            with self._get_connection() as conn:
                conn.execute(
                    """
                    INSERT INTO missions (id, topic, status, stage, created_at, updated_at, summary, total_cost)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
                    ON CONFLICT(id) DO UPDATE SET
                        status=excluded.status,
                        stage=excluded.stage,
                        updated_at=excluded.updated_at,
                        summary=COALESCE(excluded.summary, missions.summary),
                        total_cost=missions.total_cost + excluded.total_cost
                    """,
                    (mission_id, topic, status.value, stage.value, now, now, summary, total_cost),
                )
                conn.commit()

        await asyncio.to_thread(_exec)

    async def update_mission_status(self, mission_id: str, status: MissionStatus, stage: Optional[DecisionStage] = None) -> None:
        def _exec():
            now = datetime.now(timezone.utc).isoformat()
            with self._get_connection() as conn:
                if stage:
                    conn.execute("UPDATE missions SET status=?, stage=?, updated_at=? WHERE id=?", (status.value, stage.value, now, mission_id))
                else:
                    conn.execute("UPDATE missions SET status=?, updated_at=? WHERE id=?", (status.value, now, mission_id))
                conn.commit()

        await asyncio.to_thread(_exec)

    async def delete_mission(self, mission_id: str) -> None:
        def _exec():
            with self._get_connection() as conn:
                conn.execute("DELETE FROM missions WHERE id=?", (mission_id,))
                conn.execute("DELETE FROM memories WHERE mission_id=?", (mission_id,))
                conn.execute("DELETE FROM journals WHERE mission_id=?", (mission_id,))
                conn.execute("DELETE FROM messages WHERE mission_id=?", (mission_id,))
                conn.execute("DELETE FROM evidence WHERE mission_id=?", (mission_id,))
                conn.execute("DELETE FROM reflections WHERE mission_id=?", (mission_id,))
                conn.execute("DELETE FROM screen_memories WHERE mission_id=?", (mission_id,))
                conn.execute("DELETE FROM downloaded_files WHERE mission_id=?", (mission_id,))
                conn.execute("DELETE FROM checkpoints WHERE mission_id=?", (mission_id,))
                conn.execute("DELETE FROM knowledge_nodes WHERE mission_id=?", (mission_id,))
                conn.execute("DELETE FROM knowledge_edges WHERE mission_id=?", (mission_id,))
                conn.execute("DELETE FROM blackboard WHERE mission_id=?", (mission_id,))
                conn.execute("DELETE FROM mission_contexts WHERE mission_id=?", (mission_id,))
                conn.execute("DELETE FROM mission_timeline WHERE mission_id=?", (mission_id,))
                conn.execute("DELETE FROM discussions WHERE mission_id=?", (mission_id,))
                conn.execute("DELETE FROM debates WHERE mission_id=?", (mission_id,))
                conn.execute("DELETE FROM agent_reflections_v2 WHERE mission_id=?", (mission_id,))
                conn.execute("DELETE FROM approval_requests WHERE mission_id=?", (mission_id,))
                conn.execute("DELETE FROM notifications WHERE mission_id=?", (mission_id,))
                conn.commit()

        await asyncio.to_thread(_exec)

    async def get_mission(self, mission_id: str) -> Optional[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM missions WHERE id = ?", (mission_id,))
                row = cursor.fetchone()
                return dict(row) if row else None

        return await asyncio.to_thread(_exec)

    async def get_all_missions(self) -> List[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM missions ORDER BY created_at DESC")
                return [dict(row) for row in cursor.fetchall()]

        return await asyncio.to_thread(_exec)

    async def count_missions(self) -> int:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT COUNT(*) FROM missions")
                return cursor.fetchone()[0]

        return await asyncio.to_thread(_exec)

    async def upsert_agent(
        self, agent_id: str, name: str, role: str, state: AgentState, current_task: Optional[str], confidence: float, enabled: bool = True
    ) -> None:
        def _exec():
            now = datetime.now(timezone.utc).isoformat()
            with self._get_connection() as conn:
                conn.execute(
                    """
                    INSERT INTO agents (id, name, role, state, current_task, confidence, last_active, enabled)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
                    ON CONFLICT(id) DO UPDATE SET
                        name=excluded.name,
                        role=excluded.role,
                        state=excluded.state,
                        current_task=excluded.current_task,
                        confidence=excluded.confidence,
                        last_active=excluded.last_active,
                        enabled=excluded.enabled
                    """,
                    (agent_id, name, role, state.value, current_task, confidence, now, 1 if enabled else 0),
                )
                conn.commit()

        await asyncio.to_thread(_exec)

    async def get_all_agents(self) -> List[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM agents")
                return [dict(row) for row in cursor.fetchall()]

        return await asyncio.to_thread(_exec)

    async def get_agent(self, agent_id: str) -> Optional[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM agents WHERE id=?", (agent_id,))
                row = cursor.fetchone()
                return dict(row) if row else None

        return await asyncio.to_thread(_exec)

    async def save_memory(self, agent_id: str, mission_id: str, content: str, tags: List[str]) -> str:
        memory_id = str(uuid.uuid4())
        tags_str = json.dumps(tags)
        now = datetime.now(timezone.utc).isoformat()

        def _exec():
            with self._get_connection() as conn:
                conn.execute(
                    "INSERT INTO memories (id, agent_id, mission_id, content, tags, created_at) VALUES (?, ?, ?, ?, ?, ?)",
                    (memory_id, agent_id, mission_id, content, tags_str, now),
                )
                conn.commit()

        await asyncio.to_thread(_exec)
        return memory_id

    async def search_memories(self, query: str, tag: Optional[str] = None) -> List[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                if tag:
                    cursor.execute("SELECT * FROM memories WHERE content LIKE ? AND tags LIKE ?", (f"%{query}%", f"%{tag}%"))
                else:
                    cursor.execute("SELECT * FROM memories WHERE content LIKE ?", (f"%{query}%",))
                rows = cursor.fetchall()
                results = []
                for r in rows:
                    item = dict(r)
                    item["tags"] = json.loads(item["tags"]) if item["tags"] else []
                    results.append(item)
                return results

        return await asyncio.to_thread(_exec)

    async def delete_memory(self, memory_id: str) -> None:
        def _exec():
            with self._get_connection() as conn:
                conn.execute("DELETE FROM memories WHERE id=?", (memory_id,))
                conn.commit()

        await asyncio.to_thread(_exec)

    async def get_memories_for_mission(self, mission_id: str) -> List[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM memories WHERE mission_id = ? ORDER BY created_at ASC", (mission_id,))
                rows = cursor.fetchall()
                results = []
                for row in rows:
                    item = dict(row)
                    item["tags"] = json.loads(item["tags"]) if item["tags"] else []
                    results.append(item)
                return results

        return await asyncio.to_thread(_exec)

    async def count_memories(self) -> int:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT COUNT(*) FROM memories")
                return cursor.fetchone()[0]

        return await asyncio.to_thread(_exec)

    async def save_journal(self, agent_id: str, mission_id: str, entry: str) -> str:
        journal_id = str(uuid.uuid4())
        now = datetime.now(timezone.utc).isoformat()

        def _exec():
            with self._get_connection() as conn:
                conn.execute(
                    "INSERT INTO journals (id, agent_id, mission_id, entry, timestamp) VALUES (?, ?, ?, ?, ?)",
                    (journal_id, agent_id, mission_id, entry, now),
                )
                conn.commit()

        await asyncio.to_thread(_exec)
        return journal_id

    async def get_journals_for_mission(self, mission_id: str) -> List[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM journals WHERE mission_id = ? ORDER BY timestamp ASC", (mission_id,))
                return [dict(row) for row in cursor.fetchall()]

        return await asyncio.to_thread(_exec)

    async def save_message(self, message: AgentMessage) -> None:
        def _exec():
            with self._get_connection() as conn:
                conn.execute(
                    """
                    INSERT INTO messages (id, sender, recipient, mission_id, status, summary, next_request, confidence, timestamp)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
                    """,
                    (
                        message.id,
                        message.sender,
                        message.recipient,
                        message.mission_id,
                        message.status,
                        message.summary,
                        message.next_request,
                        message.confidence,
                        message.timestamp,
                    ),
                )
                conn.commit()

        await asyncio.to_thread(_exec)

    async def get_messages_for_mission(self, mission_id: str) -> List[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM messages WHERE mission_id = ? ORDER BY timestamp ASC", (mission_id,))
                return [dict(row) for row in cursor.fetchall()]

        return await asyncio.to_thread(_exec)

    async def count_messages(self) -> int:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT COUNT(*) FROM messages")
                return cursor.fetchone()[0]

        return await asyncio.to_thread(_exec)

    async def save_evidence(
        self,
        mission_id: str,
        agent_id: str,
        claim: str,
        source: str,
        score: EvidenceScore,
    ) -> str:
        evidence_id = str(uuid.uuid4())
        now = datetime.now(timezone.utc).isoformat()

        def _exec():
            with self._get_connection() as conn:
                conn.execute(
                    """
                    INSERT INTO evidence (id, mission_id, agent_id, claim, source, credibility, freshness, authority, agreement, conflict, confidence, created_at)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                    """,
                    (
                        evidence_id,
                        mission_id,
                        agent_id,
                        claim,
                        source,
                        score.credibility,
                        score.freshness,
                        score.authority,
                        score.agreement,
                        score.conflict,
                        score.overall_confidence,
                        now,
                    ),
                )
                conn.commit()

        await asyncio.to_thread(_exec)
        return evidence_id

    async def get_evidence_for_mission(self, mission_id: str) -> List[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM evidence WHERE mission_id = ? ORDER BY created_at ASC", (mission_id,))
                return [dict(row) for row in cursor.fetchall()]

        return await asyncio.to_thread(_exec)

    async def record_tokens(
        self,
        mission_id: str,
        agent_id: str,
        prompt_tokens: int,
        completion_tokens: int,
        reasoning_tokens: int,
        vision_tokens: int,
        cost_usd: float,
    ) -> None:
        token_id = str(uuid.uuid4())
        now = datetime.now(timezone.utc).isoformat()

        def _exec():
            with self._get_connection() as conn:
                conn.execute(
                    """
                    INSERT INTO token_costs (id, mission_id, agent_id, prompt_tokens, completion_tokens, reasoning_tokens, vision_tokens, cost_usd, timestamp)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
                    """,
                    (token_id, mission_id, agent_id, prompt_tokens, completion_tokens, reasoning_tokens, vision_tokens, cost_usd, now),
                )
                conn.commit()

        await asyncio.to_thread(_exec)

    async def get_total_system_cost(self) -> float:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT SUM(cost_usd) FROM token_costs")
                res = cursor.fetchone()[0]
                return res if res else 0.0

        return await asyncio.to_thread(_exec)

    async def save_browser_cache(self, url: str, html: str, title: Optional[str]) -> str:
        cache_id = str(uuid.uuid4())
        now = datetime.now(timezone.utc).isoformat()

        def _exec():
            with self._get_connection() as conn:
                conn.execute(
                    """
                    INSERT INTO browser_cache (id, url, html, title, cached_at)
                    VALUES (?, ?, ?, ?, ?)
                    ON CONFLICT(url) DO UPDATE SET html=excluded.html, title=excluded.title, cached_at=excluded.cached_at
                    """,
                    (cache_id, url, html, title, now),
                )
                conn.commit()

        await asyncio.to_thread(_exec)
        return cache_id

    async def clear_browser_cache(self) -> None:
        def _exec():
            with self._get_connection() as conn:
                conn.execute("DELETE FROM browser_cache")
                conn.commit()

        await asyncio.to_thread(_exec)

    async def save_plugin(self, name: str, version: str, description: str, entry_point: str, permissions: List[str]) -> str:
        plugin_id = str(uuid.uuid4())
        now = datetime.now(timezone.utc).isoformat()
        perms_str = json.dumps(permissions)

        def _exec():
            with self._get_connection() as conn:
                conn.execute(
                    """
                    INSERT INTO plugins (id, name, version, description, entry_point, permissions, status, registered_at)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
                    ON CONFLICT(name) DO UPDATE SET version=excluded.version, status=excluded.status
                    """,
                    (plugin_id, name, version, description, entry_point, perms_str, "ACTIVE", now),
                )
                conn.commit()

        await asyncio.to_thread(_exec)
        return plugin_id

    async def get_all_plugins(self) -> List[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM plugins")
                rows = cursor.fetchall()
                results = []
                for r in rows:
                    item = dict(r)
                    item["permissions"] = json.loads(item["permissions"]) if item["permissions"] else []
                    results.append(item)
                return results

        return await asyncio.to_thread(_exec)

    async def save_reflection(
        self,
        mission_id: str,
        agent_id: str,
        lessons_learned: str,
        mistakes_identified: Optional[str],
        cost_usd: float,
        confidence_achieved: float,
    ) -> str:
        ref_id = str(uuid.uuid4())
        now = datetime.now(timezone.utc).isoformat()

        def _exec():
            with self._get_connection() as conn:
                conn.execute(
                    """
                    INSERT INTO reflections (id, mission_id, agent_id, lessons_learned, mistakes_identified, cost_usd, confidence_achieved, created_at)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
                    """,
                    (ref_id, mission_id, agent_id, lessons_learned, mistakes_identified, cost_usd, confidence_achieved, now),
                )
                conn.commit()

        await asyncio.to_thread(_exec)
        return ref_id

    async def get_reflections_for_mission(self, mission_id: str) -> List[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM reflections WHERE mission_id = ? ORDER BY created_at ASC", (mission_id,))
                return [dict(row) for row in cursor.fetchall()]

        return await asyncio.to_thread(_exec)

    async def get_all_reflections(self) -> List[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM reflections ORDER BY created_at DESC")
                return [dict(row) for row in cursor.fetchall()]

        return await asyncio.to_thread(_exec)

    async def upsert_website_profile(
        self,
        domain: str,
        trust_score: float,
        authority: float,
        typical_layout: str,
        has_captcha: bool = False,
        success_rate: float = 100.0,
    ) -> None:
        def _exec():
            now = datetime.now(timezone.utc).isoformat()
            with self._get_connection() as conn:
                conn.execute(
                    """
                    INSERT INTO website_profiles (domain, trust_score, authority, typical_layout, has_captcha_history, interaction_success_rate, last_visited)
                    VALUES (?, ?, ?, ?, ?, ?, ?)
                    ON CONFLICT(domain) DO UPDATE SET
                        trust_score=excluded.trust_score,
                        authority=excluded.authority,
                        typical_layout=excluded.typical_layout,
                        has_captcha_history=excluded.has_captcha_history,
                        interaction_success_rate=excluded.interaction_success_rate,
                        last_visited=excluded.last_visited
                    """,
                    (domain, trust_score, authority, typical_layout, 1 if has_captcha else 0, success_rate, now),
                )
                conn.commit()

        await asyncio.to_thread(_exec)

    async def get_website_profiles(self) -> List[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM website_profiles ORDER BY trust_score DESC")
                return [dict(row) for row in cursor.fetchall()]

        return await asyncio.to_thread(_exec)

    async def save_screen_memory(self, mission_id: str, url: str, screenshot_ref: str, layout_summary: str) -> str:
        mem_id = str(uuid.uuid4())
        now = datetime.now(timezone.utc).isoformat()

        def _exec():
            with self._get_connection() as conn:
                conn.execute(
                    "INSERT INTO screen_memories (id, mission_id, url, screenshot_ref, layout_summary, timestamp) VALUES (?, ?, ?, ?, ?, ?)",
                    (mem_id, mission_id, url, screenshot_ref, layout_summary, now),
                )
                conn.commit()

        await asyncio.to_thread(_exec)
        return mem_id

    async def save_downloaded_file(self, mission_id: str, filename: str, mime_type: str, file_size: int, summary: str) -> str:
        file_id = str(uuid.uuid4())
        now = datetime.now(timezone.utc).isoformat()

        def _exec():
            with self._get_connection() as conn:
                conn.execute(
                    "INSERT INTO downloaded_files (id, mission_id, filename, mime_type, file_size, summary, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
                    (file_id, mission_id, filename, mime_type, file_size, summary, now),
                )
                conn.commit()

        await asyncio.to_thread(_exec)
        return file_id

    async def get_downloaded_files(self) -> List[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM downloaded_files ORDER BY created_at DESC")
                return [dict(row) for row in cursor.fetchall()]

        return await asyncio.to_thread(_exec)

    async def save_checkpoint(self, mission_id: str, stage: DecisionStage, data: Dict[str, Any]) -> str:
        cp_id = str(uuid.uuid4())
        now = datetime.now(timezone.utc).isoformat()
        data_str = json.dumps(data)

        def _exec():
            with self._get_connection() as conn:
                conn.execute(
                    "INSERT INTO checkpoints (id, mission_id, stage, data_json, created_at) VALUES (?, ?, ?, ?, ?)",
                    (cp_id, mission_id, stage.value, data_str, now),
                )
                conn.commit()

        await asyncio.to_thread(_exec)
        return cp_id

    async def get_latest_checkpoint(self, mission_id: str) -> Optional[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM checkpoints WHERE mission_id = ? ORDER BY created_at DESC LIMIT 1", (mission_id,))
                row = cursor.fetchone()
                if not row:
                    return None
                item = dict(row)
                item["data"] = json.loads(item["data_json"])
                return item

        return await asyncio.to_thread(_exec)

    async def get_checkpoints_for_mission(self, mission_id: str) -> List[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM checkpoints WHERE mission_id = ? ORDER BY created_at ASC", (mission_id,))
                rows = cursor.fetchall()
                results = []
                for r in rows:
                    item = dict(r)
                    item["data"] = json.loads(item["data_json"])
                    results.append(item)
                return results

        return await asyncio.to_thread(_exec)

    async def save_knowledge_node(self, node: KnowledgeNode) -> None:
        def _exec():
            with self._get_connection() as conn:
                conn.execute(
                    "INSERT INTO knowledge_nodes (id, mission_id, label, entity_type, confidence) VALUES (?, ?, ?, ?, ?)",
                    (node.id, node.mission_id, node.label, node.entity_type, node.confidence),
                )
                conn.commit()

        await asyncio.to_thread(_exec)

    async def save_knowledge_edge(self, edge: KnowledgeEdge) -> None:
        def _exec():
            with self._get_connection() as conn:
                conn.execute(
                    "INSERT INTO knowledge_edges (id, mission_id, source_node_id, target_node_id, relationship) VALUES (?, ?, ?, ?, ?)",
                    (edge.id, edge.mission_id, edge.source_node_id, edge.target_node_id, edge.relationship),
                )
                conn.commit()

        await asyncio.to_thread(_exec)

    async def get_knowledge_graph(self, mission_id: str) -> Dict[str, Any]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM knowledge_nodes WHERE mission_id=?", (mission_id,))
                nodes = [dict(r) for r in cursor.fetchall()]
                cursor.execute("SELECT * FROM knowledge_edges WHERE mission_id=?", (mission_id,))
                edges = [dict(r) for r in cursor.fetchall()]
                return {"mission_id": mission_id, "nodes": nodes, "edges": edges}

        return await asyncio.to_thread(_exec)

    async def save_blackboard_entry(self, entry: BlackboardEntry) -> None:
        def _exec():
            with self._get_connection() as conn:
                conn.execute(
                    "INSERT INTO blackboard (id, mission_id, agent_id, topic, data_json, timestamp) VALUES (?, ?, ?, ?, ?, ?)",
                    (entry.id, entry.mission_id, entry.agent_id, entry.topic, json.dumps(entry.data), entry.timestamp),
                )
                conn.commit()

        await asyncio.to_thread(_exec)

    async def get_blackboard_entries(self, mission_id: str, topic: Optional[str] = None) -> List[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                if topic:
                    cursor.execute("SELECT * FROM blackboard WHERE mission_id = ? AND topic = ? ORDER BY timestamp ASC", (mission_id, topic))
                else:
                    cursor.execute("SELECT * FROM blackboard WHERE mission_id = ? ORDER BY timestamp ASC", (mission_id,))
                rows = cursor.fetchall()
                results = []
                for r in rows:
                    item = dict(r)
                    item["data"] = json.loads(item["data_json"])
                    results.append(item)
                return results

        return await asyncio.to_thread(_exec)

    async def save_mission_context(self, ctx: MissionContextModel) -> None:
        def _exec():
            with self._get_connection() as conn:
                conn.execute(
                    """
                    INSERT INTO mission_contexts 
                    (mission_id, topic, priority, current_phase, progress_percent, owner_agent, resource_usage_json, estimated_cost, health_status, retry_count, checkpoint_state, created_at)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                    ON CONFLICT(mission_id) DO UPDATE SET
                        current_phase=excluded.current_phase,
                        progress_percent=excluded.progress_percent,
                        resource_usage_json=excluded.resource_usage_json,
                        estimated_cost=excluded.estimated_cost,
                        health_status=excluded.health_status,
                        retry_count=excluded.retry_count,
                        checkpoint_state=excluded.checkpoint_state
                    """,
                    (
                        ctx.mission_id,
                        ctx.topic,
                        ctx.priority,
                        ctx.current_phase.value,
                        ctx.progress_percent,
                        ctx.owner_agent,
                        json.dumps(ctx.resource_usage),
                        ctx.estimated_cost,
                        ctx.health_status,
                        ctx.retry_count,
                        ctx.checkpoint_state,
                        ctx.created_at,
                    ),
                )
                conn.commit()

        await asyncio.to_thread(_exec)

    async def get_all_mission_contexts(self) -> List[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM mission_contexts ORDER BY created_at DESC")
                rows = cursor.fetchall()
                results = []
                for r in rows:
                    item = dict(r)
                    item["resource_usage"] = json.loads(item["resource_usage_json"])
                    results.append(item)
                return results

        return await asyncio.to_thread(_exec)

    async def save_timeline_event(self, event: TimelineEventModel) -> None:
        def _exec():
            with self._get_connection() as conn:
                conn.execute(
                    "INSERT INTO mission_timeline (id, mission_id, agent_id, step_type, description, metadata_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?)",
                    (event.id, event.mission_id, event.agent_id, event.step_type.value, event.description, json.dumps(event.metadata), event.timestamp),
                )
                conn.commit()

        await asyncio.to_thread(_exec)

    async def get_mission_timeline(self, mission_id: str) -> List[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM mission_timeline WHERE mission_id = ? ORDER BY timestamp ASC", (mission_id,))
                rows = cursor.fetchall()
                results = []
                for r in rows:
                    item = dict(r)
                    item["metadata"] = json.loads(item["metadata_json"])
                    results.append(item)
                return results

        return await asyncio.to_thread(_exec)

    async def save_discussion_entry(self, entry: DiscussionEntry) -> None:
        def _exec():
            with self._get_connection() as conn:
                conn.execute(
                    """
                    INSERT INTO discussions (id, mission_id, agent_id, agent_name, discussion_type, topic, content, evidence_ref, tags_json, confidence, timestamp)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                    """,
                    (
                        entry.id,
                        entry.mission_id,
                        entry.agent_id,
                        entry.agent_name,
                        entry.discussion_type.value,
                        entry.topic,
                        entry.content,
                        entry.evidence_ref,
                        json.dumps(entry.tags),
                        entry.confidence,
                        entry.timestamp,
                    ),
                )
                conn.commit()

        await asyncio.to_thread(_exec)

    async def get_discussions_for_mission(self, mission_id: str, tag: Optional[str] = None) -> List[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                if tag:
                    cursor.execute("SELECT * FROM discussions WHERE mission_id = ? AND tags_json LIKE ? ORDER BY timestamp ASC", (mission_id, f"%{tag}%"))
                else:
                    cursor.execute("SELECT * FROM discussions WHERE mission_id = ? ORDER BY timestamp ASC", (mission_id,))
                rows = cursor.fetchall()
                results = []
                for r in rows:
                    item = dict(r)
                    item["tags"] = json.loads(item["tags_json"]) if item.get("tags_json") else []
                    results.append(item)
                return results

        return await asyncio.to_thread(_exec)

    async def save_debate_session(self, debate: DebateSession) -> None:
        def _exec():
            with self._get_connection() as conn:
                turns_json = json.dumps([t.model_dump() for t in debate.turns])
                conn.execute(
                    """
                    INSERT INTO debates (debate_id, mission_id, topic, status, participants_json, turns_json, consensus_summary, final_confidence, created_at)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
                    ON CONFLICT(debate_id) DO UPDATE SET
                        status=excluded.status,
                        turns_json=excluded.turns_json,
                        consensus_summary=excluded.consensus_summary,
                        final_confidence=excluded.final_confidence
                    """,
                    (
                        debate.debate_id,
                        debate.mission_id,
                        debate.topic,
                        debate.status,
                        json.dumps(debate.participants),
                        turns_json,
                        debate.consensus_summary,
                        debate.final_confidence,
                        debate.created_at,
                    ),
                )
                conn.commit()

        await asyncio.to_thread(_exec)

    async def get_debate_session(self, debate_id: str) -> Optional[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM debates WHERE debate_id = ?", (debate_id,))
                row = cursor.fetchone()
                if not row:
                    return None
                item = dict(row)
                item["participants"] = json.loads(item["participants_json"])
                item["turns"] = json.loads(item["turns_json"])
                return item

        return await asyncio.to_thread(_exec)

    async def get_debates_for_mission(self, mission_id: str) -> List[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM debates WHERE mission_id = ? ORDER BY created_at ASC", (mission_id,))
                rows = cursor.fetchall()
                results = []
                for r in rows:
                    item = dict(r)
                    item["participants"] = json.loads(item["participants_json"])
                    item["turns"] = json.loads(item["turns_json"])
                    results.append(item)
                return results

        return await asyncio.to_thread(_exec)

    async def save_agent_reputation(self, rep: AgentReputationModel) -> None:
        def _exec():
            with self._get_connection() as conn:
                conn.execute(
                    """
                    INSERT INTO agent_reputations 
                    (agent_id, name, role, experience_points, trust_score, accuracy_rate, reliability_score, speed_score, cost_efficiency, avg_confidence, success_rate, total_missions_participated)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                    ON CONFLICT(agent_id) DO UPDATE SET
                        experience_points=excluded.experience_points,
                        trust_score=excluded.trust_score,
                        accuracy_rate=excluded.accuracy_rate,
                        reliability_score=excluded.reliability_score,
                        speed_score=excluded.speed_score,
                        cost_efficiency=excluded.cost_efficiency,
                        avg_confidence=excluded.avg_confidence,
                        success_rate=excluded.success_rate,
                        total_missions_participated=excluded.total_missions_participated
                    """,
                    (
                        rep.agent_id,
                        rep.name,
                        rep.role,
                        rep.experience_points,
                        rep.trust_score,
                        rep.accuracy_rate,
                        rep.reliability_score,
                        rep.speed_score,
                        rep.cost_efficiency,
                        rep.avg_confidence,
                        rep.success_rate,
                        rep.total_missions_participated,
                    ),
                )
                conn.commit()

        await asyncio.to_thread(_exec)

    async def get_agent_reputation(self, agent_id: str) -> Optional[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM agent_reputations WHERE agent_id = ?", (agent_id,))
                row = cursor.fetchone()
                return dict(row) if row else None

        return await asyncio.to_thread(_exec)

    async def get_all_agent_reputations(self) -> List[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM agent_reputations ORDER BY trust_score DESC")
                return [dict(r) for r in cursor.fetchall()]

        return await asyncio.to_thread(_exec)

    async def save_agent_reflection_v2(self, refl: AgentReflectionDetail) -> None:
        def _exec():
            with self._get_connection() as conn:
                conn.execute(
                    """
                    INSERT INTO agent_reflections_v2 (reflection_id, mission_id, agent_id, agent_name, what_worked, what_failed, what_surprised, what_to_improve, timestamp)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
                    """,
                    (
                        refl.reflection_id,
                        refl.mission_id,
                        refl.agent_id,
                        refl.agent_name,
                        refl.what_worked,
                        refl.what_failed,
                        refl.what_surprised,
                        refl.what_to_improve,
                        refl.timestamp,
                    ),
                )
                conn.commit()

        await asyncio.to_thread(_exec)

    async def get_agent_reflections_for_mission(self, mission_id: str) -> List[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM agent_reflections_v2 WHERE mission_id = ? ORDER BY timestamp ASC", (mission_id,))
                return [dict(r) for r in cursor.fetchall()]

        return await asyncio.to_thread(_exec)

    async def save_approval_request(self, req: ApprovalRequestModel) -> None:
        def _exec():
            with self._get_connection() as conn:
                conn.execute(
                    """
                    INSERT INTO approval_requests (id, mission_id, agent_id, action_type, prompt_message, status, input_data_json, created_at)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
                    ON CONFLICT(id) DO UPDATE SET status=excluded.status, input_data_json=excluded.input_data_json
                    """,
                    (req.id, req.mission_id, req.agent_id, req.action_type.value, req.prompt_message, req.status.value, json.dumps(req.input_data), req.created_at),
                )
                conn.commit()

        await asyncio.to_thread(_exec)

    async def get_approval_request(self, approval_id: str) -> Optional[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM approval_requests WHERE id = ?", (approval_id,))
                row = cursor.fetchone()
                if not row:
                    return None
                item = dict(row)
                item["input_data"] = json.loads(item["input_data_json"]) if item.get("input_data_json") else {}
                return item

        return await asyncio.to_thread(_exec)

    async def get_pending_approvals(self) -> List[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM approval_requests WHERE status = 'PENDING' ORDER BY created_at ASC")
                rows = cursor.fetchall()
                results = []
                for r in rows:
                    item = dict(r)
                    item["input_data"] = json.loads(item["input_data_json"]) if item.get("input_data_json") else {}
                    results.append(item)
                return results

        return await asyncio.to_thread(_exec)

    async def save_notification(self, notif: NotificationModel) -> None:
        def _exec():
            with self._get_connection() as conn:
                conn.execute(
                    "INSERT INTO notifications (id, level, title, message, mission_id, acknowledged, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
                    (notif.id, notif.level.value, notif.title, notif.message, notif.mission_id, 1 if notif.acknowledged else 0, notif.created_at),
                )
                conn.commit()

        await asyncio.to_thread(_exec)

    async def get_unacknowledged_notifications(self) -> List[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM notifications WHERE acknowledged = 0 ORDER BY created_at DESC")
                return [dict(r) for r in cursor.fetchall()]

        return await asyncio.to_thread(_exec)

    async def acknowledge_notification(self, notif_id: str) -> None:
        def _exec():
            with self._get_connection() as conn:
                conn.execute("UPDATE notifications SET acknowledged = 1 WHERE id = ?", (notif_id,))
                conn.commit()

        await asyncio.to_thread(_exec)

    async def save_conversation_log(self, msg: ConversationMessageModel) -> None:
        def _exec():
            with self._get_connection() as conn:
                conn.execute(
                    "INSERT INTO conversation_logs (id, user_id, sender, message, metadata_json, timestamp) VALUES (?, ?, ?, ?, ?, ?)",
                    (msg.id, msg.user_id, msg.sender, msg.message, json.dumps(msg.metadata), msg.timestamp),
                )
                conn.commit()

        await asyncio.to_thread(_exec)

    async def get_conversation_history(self, user_id: str = "human-operator", limit: int = 50) -> List[Dict[str, Any]]:
        def _exec():
            with self._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT * FROM conversation_logs WHERE user_id = ? ORDER BY timestamp ASC LIMIT ?", (user_id, limit))
                rows = cursor.fetchall()
                results = []
                for r in rows:
                    item = dict(r)
                    item["metadata"] = json.loads(item["metadata_json"]) if item.get("metadata_json") else {}
                    results.append(item)
                return results

        return await asyncio.to_thread(_exec)


# --- WEBSOCKET CONNECTION MANAGER ---
class WebSocketManager:
    """Manages active WebSocket client connections for real-time live events."""

    def __init__(self):
        self.active_connections: Set[WebSocket] = set()

    async def connect(self, websocket: WebSocket):
        await websocket.accept()
        self.active_connections.add(websocket)
        logger.info(f"WebSocket client connected. Total clients: {len(self.active_connections)}")

    def disconnect(self, websocket: WebSocket):
        self.active_connections.discard(websocket)
        logger.info(f"WebSocket client disconnected. Total clients: {len(self.active_connections)}")

    async def broadcast(self, payload: Dict[str, Any]):
        if not self.active_connections:
            return
        stale = set()
        for conn in list(self.active_connections):
            try:
                await conn.send_json(payload)
            except Exception:
                stale.add(conn)
        for conn in stale:
            self.disconnect(conn)


# --- EVENT BUS & MESSAGE BUS ENGINE ---
class EventBus:
    """Central Real-time Live Event Streaming Queue."""

    def __init__(self, ws_manager: WebSocketManager):
        self.ws_manager = ws_manager

    async def emit(self, event_type: str, mission_id: str, agent_name: str, data: Dict[str, Any]) -> None:
        payload = {
            "event_type": event_type,
            "mission_id": mission_id,
            "agent_name": agent_name,
            "data": data,
            "timestamp": datetime.now(timezone.utc).isoformat(),
        }
        logger.info(f"LIVE EVENT [{event_type}] | Mission: {mission_id} | Agent: {agent_name}")
        await self.ws_manager.broadcast(payload)


class MessageBus:
    """Central Agent Communication Queue enforcing structured message flow."""

    def __init__(self, db: DatabaseManager, event_bus: EventBus):
        self.db = db
        self.event_bus = event_bus

    async def publish(self, message: AgentMessage) -> None:
        await self.db.save_message(message)
        await self.event_bus.emit(
            event_type="AgentMessageSent",
            mission_id=message.mission_id,
            agent_name=message.sender,
            data=message.model_dump(),
        )


# --- BLACKBOARD & RESOURCE MANAGEMENT ENGINES ---
class ColonyBlackboard:
    """Shared memory board for real-time observation publishing and cross-agent synchronization."""

    def __init__(self, db: DatabaseManager, event_bus: EventBus):
        self.db = db
        self.event_bus = event_bus
        self._entries: List[BlackboardEntry] = []

    async def publish(self, mission_id: str, agent_id: str, topic: str, data: Dict[str, Any]) -> BlackboardEntry:
        entry = BlackboardEntry(mission_id=mission_id, agent_id=agent_id, topic=topic, data=data)
        self._entries.append(entry)
        await self.db.save_blackboard_entry(entry)
        await self.event_bus.emit(
            event_type="BlackboardUpdated",
            mission_id=mission_id,
            agent_name=agent_id,
            data={"topic": topic, "entry_id": entry.id, "summary": str(data)[:100]},
        )
        return entry

    async def query(self, mission_id: str, topic: Optional[str] = None) -> List[Dict[str, Any]]:
        return await self.db.get_blackboard_entries(mission_id, topic)


class ColonyResourceManager:
    """Tracks system hardware, LLM tokens (Gemini/Groq), database load, and queue pressure."""

    def __init__(self):
        self.gemini_tokens: int = 0
        self.groq_tokens: int = 0
        self.sqlite_queries: int = 0
        self.vector_searches: int = 0

    def record_gemini_tokens(self, tokens: int):
        self.gemini_tokens += tokens

    def record_groq_tokens(self, tokens: int):
        self.groq_tokens += tokens

    def record_sqlite_query(self):
        self.sqlite_queries += 1

    def record_vector_search(self):
        self.vector_searches += 1

    def get_resource_metrics(self, browser_pool: "BrowserPoolManager", scheduler: "TaskScheduler") -> ResourceMetricsModel:
        cpu_pct = psutil.cpu_percent(interval=None) if psutil else 0.0
        mem_pct = psutil.virtual_memory().percent if psutil else 0.0
        pool_status = browser_pool.get_status()

        return ResourceMetricsModel(
            cpu_usage_percent=cpu_pct,
            memory_usage_percent=mem_pct,
            browser_instances=pool_status["total_browsers"],
            open_tabs=pool_status["active_browsers"],
            gemini_tokens_used=self.gemini_tokens,
            groq_tokens_used=self.groq_tokens,
            sqlite_queries_count=self.sqlite_queries,
            vector_searches_count=self.vector_searches,
            task_queue_depth=scheduler.queue_size(),
        )


# --- API KEY ROTATION & UNIVERSAL MODEL MANAGER ---
class ManagedKey:
    def __init__(self, key_id: str, provider: str, secret: str):
        self.key_id = key_id
        self.provider = provider
        self.secret = secret
        self.is_busy = False
        self.is_disabled = False
        self.error_count = 0
        self.total_calls = 0
        self.cooldown_until = 0.0


class APIKeyRotationEngine:
    """Manages dynamic key rotation, concurrency isolation, rate-limit backoff, and recovery."""

    def __init__(self):
        self.keys: List[ManagedKey] = []
        self._load_keys_from_env()

    def _load_keys_from_env(self):
        # Load Gemini keys
        gemini_keys_str = os.getenv("GEMINI_KEYS", os.getenv("GEMINI_API_KEY", ""))
        raw_g_keys = [k.strip() for k in gemini_keys_str.split(",") if k.strip()]
        for idx, k in enumerate(raw_g_keys):
            self.keys.append(ManagedKey(f"gemini-key-{idx+1}", "gemini", k))

        for env_k, env_v in os.environ.items():
            if env_k.startswith("GEMINI_API_KEY_") and env_v.strip():
                if not any(mk.secret == env_v.strip() for mk in self.keys):
                    self.keys.append(ManagedKey(f"gemini-{env_k.lower()}", "gemini", env_v.strip()))

        # Load Groq keys
        groq_keys_str = os.getenv("GROQ_KEYS", os.getenv("GROQ_API_KEY", ""))
        raw_q_keys = [k.strip() for k in groq_keys_str.split(",") if k.strip()]
        for idx, k in enumerate(raw_q_keys):
            self.keys.append(ManagedKey(f"groq-key-{idx+1}", "groq", k))

        for env_k, env_v in os.environ.items():
            if env_k.startswith("GROQ_API_KEY_") and env_v.strip():
                if not any(mk.secret == env_v.strip() for mk in self.keys):
                    self.keys.append(ManagedKey(f"groq-{env_k.lower()}", "groq", env_v.strip()))

        if not self.keys:
            # Add fallback dev keys if no env keys are present
            self.keys.append(ManagedKey("fallback-gemini-dev", "gemini", "DEV_MODE_GEMINI_KEY"))
            self.keys.append(ManagedKey("fallback-groq-dev", "groq", "DEV_MODE_GROQ_KEY"))

    def acquire_key(self, provider: str = "gemini") -> Optional[ManagedKey]:
        now = time.time()
        for k in self.keys:
            if k.provider == provider and not k.is_busy and not k.is_disabled:
                if k.cooldown_until > now:
                    continue
                k.is_busy = True
                k.total_calls += 1
                return k
        # Auto-recover disabled keys if cool-down passed
        for k in self.keys:
            if k.provider == provider and k.is_disabled and k.cooldown_until <= now:
                k.is_disabled = False
                k.error_count = 0
                k.is_busy = True
                k.total_calls += 1
                return k
        return None

    def release_key(self, key: ManagedKey, success: bool = True, error_msg: str = ""):
        key.is_busy = False
        now = time.time()
        if success:
            key.error_count = max(0, key.error_count - 1)
        else:
            key.error_count += 1
            if "429" in error_msg or "rate limit" in error_msg.lower():
                key.cooldown_until = now + 60.0  # 1 min cooldown for rate limits
            else:
                key.cooldown_until = now + 15.0
            if key.error_count >= 5:
                key.is_disabled = True
                key.cooldown_until = now + 300.0  # 5 min suspension for failing key

    def get_telemetry(self) -> List[KeyStatusModel]:
        now = time.time()
        return [
            KeyStatusModel(
                key_id=k.key_id,
                provider=k.provider,
                is_busy=k.is_busy,
                is_disabled=k.is_disabled,
                error_count=k.error_count,
                total_calls=k.total_calls,
                cooldown_remaining_sec=max(0.0, round(k.cooldown_until - now, 1)),
            )
            for k in self.keys
        ]


class ModelManager:
    """Universal Model Gateway mapping logical models (mdl_fst, mdl_adv) to dynamic physical models."""

    def __init__(self, key_rotator: APIKeyRotationEngine, resource_manager: ColonyResourceManager):
        self.key_rotator = key_rotator
        self.resource_manager = resource_manager
        # Model mappings loaded from environment variable with fallbacks
        self.logical_models = {
            LogicalModel.MDL_FST.value: os.getenv("MDL_FST", "gemini-2.5-flash"),
            LogicalModel.MDL_ADV.value: os.getenv("MDL_ADV", "llama-3.3-70b-versatile"),
        }
        self.total_calls = 0

    async def generate_response(
        self,
        logical_model: LogicalModel,
        prompt: str,
        system_prompt: Optional[str] = None,
        vision_input: Optional[str] = None,
    ) -> Dict[str, Any]:
        self.total_calls += 1
        physical_model = self.logical_models.get(logical_model.value, "gemini-2.5-flash")
        provider = "groq" if "llama" in physical_model.lower() or "groq" in physical_model.lower() else "gemini"

        key = self.key_rotator.acquire_key(provider)
        if not key:
            # Fallback to any provider if primary unavailable
            provider = "gemini" if provider == "groq" else "groq"
            key = self.key_rotator.acquire_key(provider)

        try:
            await asyncio.sleep(0.05)  # Simulated LLM latency execution
            tokens_used = len(prompt.split()) + random.randint(50, 150)

            if provider == "gemini":
                self.resource_manager.record_gemini_tokens(tokens_used)
            else:
                self.resource_manager.record_groq_tokens(tokens_used)

            response_text = f"Synthesized Response via [{logical_model.value} -> {physical_model}]: Processed input prompt ({len(prompt)} chars)."

            if key:
                self.key_rotator.release_key(key, success=True)

            return {
                "logical_model": logical_model.value,
                "physical_model": physical_model,
                "provider": provider,
                "key_used": key.key_id if key else "unauthenticated",
                "tokens_used": tokens_used,
                "content": response_text,
            }
        except Exception as e:
            if key:
                self.key_rotator.release_key(key, success=False, error_msg=str(e))
            raise e


# --- AUTONOMOUS TOOL PERMISSION ENGINE ---
class ToolPermissionEngine:
    """Manages catalog of colony tools and validates execution permissions against safety guardrails."""

    def __init__(self):
        self.catalog: Dict[str, ToolDefinition] = {
            "Playwright": ToolDefinition(
                name="Playwright", category="Browser", description="Automated web browser navigation and interaction", permission_level=SafetyLevel.INTERACTIVE_FORM
            ),
            "HTTP": ToolDefinition(
                name="HTTP", category="Network", description="Send HTTP/REST requests to external endpoints", permission_level=SafetyLevel.READ_ONLY
            ),
            "Vision": ToolDefinition(
                name="Vision", category="Perception", description="Visual OCR and screen element analysis", permission_level=SafetyLevel.READ_ONLY
            ),
            "SQLite": ToolDefinition(
                name="SQLite", category="Database", description="Read and store local structured operational data", permission_level=SafetyLevel.READ_ONLY
            ),
            "Memory": ToolDefinition(
                name="Memory", category="Memory", description="Search and persist vector/text memories", permission_level=SafetyLevel.READ_ONLY
            ),
            "Embeddings": ToolDefinition(
                name="Embeddings", category="AI", description="Generate semantic vector embeddings", permission_level=SafetyLevel.READ_ONLY
            ),
            "Filesystem": ToolDefinition(
                name="Filesystem", category="System", description="Read and write safe workspace files", permission_level=SafetyLevel.SENSITIVE_ACTION
            ),
            "Search": ToolDefinition(
                name="Search", category="Research", description="Execute web search and citation retrieval", permission_level=SafetyLevel.READ_ONLY
            ),
        }

    def evaluate_request(self, req: ToolRequest) -> ToolResponse:
        tool = self.catalog.get(req.tool_name)
        if not tool:
            return ToolResponse(tool_name=req.tool_name, allowed=False, reason=f"Tool '{req.tool_name}' not registered in catalog.")

        if not tool.enabled:
            return ToolResponse(tool_name=req.tool_name, allowed=False, reason=f"Tool '{req.tool_name}' is currently disabled.")

        if tool.permission_level == SafetyLevel.DESTRUCTIVE_BLOCKED:
            return ToolResponse(tool_name=req.tool_name, allowed=False, reason=f"Action blocked by safety policy: Destructive operations prohibited.")

        return ToolResponse(tool_name=req.tool_name, allowed=True, reason="Permission granted.", result={"status": "READY"})


# --- V2 PHASE 3 COLLABORATION & DEBATE ENGINES ---
class ColonyDiscussionEngine:
    """Manages cross-agent discussion broadcasting, evidence sharing, criticism, and memory indexing."""

    def __init__(self, db: DatabaseManager, event_bus: EventBus):
        self.db = db
        self.event_bus = event_bus

    async def post_discussion(
        self,
        mission_id: str,
        agent_id: str,
        agent_name: str,
        discussion_type: DiscussionType,
        topic: str,
        content: str,
        evidence_ref: Optional[str] = None,
        tags: Optional[List[str]] = None,
        confidence: float = 100.0,
    ) -> DiscussionEntry:
        entry = DiscussionEntry(
            mission_id=mission_id,
            agent_id=agent_id,
            agent_name=agent_name,
            discussion_type=discussion_type,
            topic=topic,
            content=content,
            evidence_ref=evidence_ref,
            tags=tags or ["discussion", discussion_type.value.lower()],
            confidence=confidence,
        )
        await self.db.save_discussion_entry(entry)
        await self.db.save_memory(agent_id, mission_id, f"[{discussion_type.value}] {agent_name}: {content}", entry.tags)
        await self.event_bus.emit(
            "AgentDiscussion",
            mission_id,
            agent_name,
            {"type": discussion_type.value, "topic": topic, "content": content[:120], "confidence": confidence},
        )
        return entry

    async def get_discussions(self, mission_id: str, tag: Optional[str] = None) -> List[Dict[str, Any]]:
        return await self.db.get_discussions_for_mission(mission_id, tag)


class ConsensusEngine:
    """Calculates weighted composite consensus EV across 7 key intelligence vectors."""

    @staticmethod
    def calculate_consensus(req: ConsensusRequest) -> ConsensusResult:
        weights = {
            "evidence": 0.25,
            "agreement": 0.20,
            "source_quality": 0.15,
            "historical_accuracy": 0.15,
            "memory_similarity": 0.10,
            "model_confidence": 0.10,
            "risk_penalty": 0.05,
        }

        weighted_score = (
            (req.evidence_confidence * weights["evidence"])
            + (req.agreement_score * weights["agreement"])
            + (req.source_quality * weights["source_quality"])
            + (req.historical_accuracy * weights["historical_accuracy"])
            + (req.memory_similarity * weights["memory_similarity"])
            + (req.model_confidence * weights["model_confidence"])
            - (req.mission_risk * weights["risk_penalty"])
        )

        composite = round(max(0.0, min(100.0, weighted_score)), 2)

        if composite >= 85.0:
            rec = "HIGH_CONFIDENCE_EXECUTE"
            tier = "TIER_1_OPTIMAL"
            risk_desc = "Low operational risk; verified across independent evidence channels."
        elif composite >= 65.0:
            rec = "PROCEED_WITH_VERIFICATION"
            tier = "TIER_2_MODERATE"
            risk_desc = "Moderate confidence; minor conflicts or unverified secondary claims."
        else:
            rec = "REQUIRES_HUMAN_REVIEW_OR_DEEPER_SEARCH"
            tier = "TIER_3_ELEVATED_RISK"
            risk_desc = "Elevated risk; high conflict score or low source authority."

        return ConsensusResult(
            mission_id=req.mission_id,
            topic=req.topic,
            composite_consensus_score=composite,
            decision_recommendation=rec,
            risk_assessment=risk_desc,
            confidence_tier=tier,
        )


class DebateEngine:
    """Orchestrates structured multi-agent debates and synthesizes consensus."""

    def __init__(self, db: DatabaseManager, event_bus: EventBus, model_manager: ModelManager):
        self.db = db
        self.event_bus = event_bus
        self.model_manager = model_manager

    async def initiate_debate(self, mission_id: str, topic: str, participants: List[Dict[str, str]]) -> DebateSession:
        debate = DebateSession(
            mission_id=mission_id,
            topic=topic,
            participants=[p["name"] for p in participants],
        )

        turns = []
        for idx, p in enumerate(participants):
            position = "SUPPORT" if idx % 2 == 0 else "CRITIQUE"
            arg_prompt = f"Provide a {position} perspective on topic '{topic}' based on available evidence."
            resp = await self.model_manager.generate_response(LogicalModel.MDL_FST, arg_prompt)

            turn = DebateTurn(
                turn_number=idx + 1,
                agent_id=p["id"],
                agent_name=p["name"],
                position=position,
                argument=f"[{position}] {resp['content']}",
                evidence_claims=[f"Claim_{idx+1} for {topic}"],
                confidence=85.0 + (idx * 2.5),
            )
            turns.append(turn)

        debate.turns = turns
        debate.status = "CONSENSUS_REACHED"
        debate.consensus_summary = f"Multi-agent debate concluded for '{topic}'. Strong alignment achieved on core evidence claims."
        debate.final_confidence = 91.5

        await self.db.save_debate_session(debate)
        await self.db.save_memory(
            "DebateEngine",
            mission_id,
            f"Debate Summary for {topic}: {debate.consensus_summary}",
            ["debate", "consensus", "collective_memory"],
        )
        await self.event_bus.emit("DebateConcluded", mission_id, "DebateEngine", {"topic": topic, "confidence": debate.final_confidence})
        return debate


class TaskNegotiationEngine:
    """Evaluates proposed agent tasks to eliminate duplicated work and optimize resource allocation."""

    def __init__(self, db: DatabaseManager, event_bus: EventBus):
        self.db = db
        self.event_bus = event_bus

    async def evaluate_proposal(self, proposal: TaskNegotiationProposal) -> TaskNegotiationResult:
        # Search memory to see if task was already completed
        existing_memories = await self.db.search_memories(proposal.task_description)
        is_duplicate = len(existing_memories) > 0

        if is_duplicate and proposal.proposed_action != "SKIP_DUPLICATE":
            res = TaskNegotiationResult(
                proposal_id=proposal.proposal_id,
                accepted=True,
                assigned_agent=proposal.proposing_agent,
                resolution_notes="Duplicate work detected in memory vault. Task skipped to conserve budget.",
            )
        else:
            res = TaskNegotiationResult(
                proposal_id=proposal.proposal_id,
                accepted=True,
                assigned_agent=proposal.proposing_agent,
                resolution_notes="Task negotiation approved. Proceeding with execution.",
            )

        await self.event_bus.emit(
            "TaskNegotiated", proposal.mission_id, proposal.proposing_agent, {"proposal_id": proposal.proposal_id, "notes": res.resolution_notes}
        )
        return res


class AgentReputationEngine:
    """Tracks and updates agent experience points, trust scores, reliability, and accuracy."""

    def __init__(self, db: DatabaseManager):
        self.db = db

    async def record_mission_outcome(self, agent_id: str, name: str, role: str, success: bool, duration_sec: float, confidence: float):
        existing = await self.db.get_agent_reputation(agent_id)
        if existing:
            rep = AgentReputationModel(**existing)
        else:
            rep = AgentReputationModel(agent_id=agent_id, name=name, role=role)

        rep.total_missions_participated += 1
        rep.experience_points += 25 if success else 5

        # Update rolling rates
        alpha = 0.2
        target_succ = 100.0 if success else 0.0
        rep.success_rate = round((1 - alpha) * rep.success_rate + alpha * target_succ, 2)
        rep.trust_score = round((rep.success_rate * 0.6) + (rep.accuracy_rate * 0.4), 2)
        rep.avg_confidence = round((1 - alpha) * rep.avg_confidence + alpha * confidence, 2)

        await self.db.save_agent_reputation(rep)
        return rep


# --- V2 PHASE 4 NOTIFICATION ENGINE ---
class NotificationEngine:
    """Central notification center managing persisted system alerts and WS events."""

    def __init__(self, db: DatabaseManager, event_bus: EventBus):
        self.db = db
        self.event_bus = event_bus

    async def notify(self, level: NotificationLevel, title: str, message: str, mission_id: Optional[str] = None) -> NotificationModel:
        notif = NotificationModel(level=level, title=title, message=message, mission_id=mission_id)
        await self.db.save_notification(notif)
        await self.event_bus.emit("NotificationCreated", mission_id or "system", "NotificationEngine", notif.model_dump())
        return notif


# --- CONFIDENCE ENGINE & COST TRACKER ---
class ConfidenceEngine:
    """Calculates objective confidence metrics based on evidence attributes."""

    @staticmethod
    def calculate_confidence(
        credibility: float, freshness: float, authority: float, agreement: float, conflict: float
    ) -> EvidenceScore:
        weighted_score = (credibility * 0.30) + (freshness * 0.20) + (authority * 0.25) + (agreement * 0.25)
        penalty = conflict * 0.35
        overall = max(0.0, min(100.0, weighted_score - penalty))

        return EvidenceScore(
            credibility=credibility,
            freshness=freshness,
            authority=authority,
            agreement=agreement,
            conflict=conflict,
            overall_confidence=round(overall, 2),
        )


class TokenCostEngine:
    """Estimates and records operational token usage and cost metrics."""

    RATES = {
        "prompt": 0.000001,
        "completion": 0.000002,
        "reasoning": 0.000003,
        "vision": 0.000005,
    }

    @classmethod
    async def track_usage(
        cls,
        db: DatabaseManager,
        mission_id: str,
        agent_id: str,
        prompt_tokens: int = 0,
        completion_tokens: int = 0,
        reasoning_tokens: int = 0,
        vision_tokens: int = 0,
    ) -> float:
        cost = (
            (prompt_tokens * cls.RATES["prompt"])
            + (completion_tokens * cls.RATES["completion"])
            + (reasoning_tokens * cls.RATES["reasoning"])
            + (vision_tokens * cls.RATES["vision"])
        )
        await db.record_tokens(
            mission_id=mission_id,
            agent_id=agent_id,
            prompt_tokens=prompt_tokens,
            completion_tokens=completion_tokens,
            reasoning_tokens=reasoning_tokens,
            vision_tokens=vision_tokens,
            cost_usd=cost,
        )
        await db.save_mission(
            mission_id=mission_id, topic="", status=MissionStatus.IN_PROGRESS, stage=DecisionStage.EXECUTE, total_cost=cost
        )
        return cost


# --- PRODUCTION REASONING & QUALITY ENGINES ---
class EvidencePyramidEngine:
    """Ranks evidence sources into hierarchical trust tiers."""

    @staticmethod
    def classify_source(source_url: str) -> SourceTier:
        url_lower = source_url.lower()
        if any(domain in url_lower for domain in [".gov", ".edu", "arxiv.org", "nature.com", "doi.org", "ncbi.nlm.nih.gov"]):
            return SourceTier.TIER_1_OFFICIAL
        elif any(domain in url_lower for domain in ["reuters.com", "apnews.com", "bbc.com", "bloomberg.com", "wsj.com"]):
            return SourceTier.TIER_2_NEWS
        elif any(domain in url_lower for domain in ["wikipedia.org", "github.com", "medium.com"]):
            return SourceTier.TIER_3_COMMUNITY
        elif any(domain in url_lower for domain in ["reddit.com", "quora.com", "stackoverflow.com"]):
            return SourceTier.TIER_4_FORUMS
        else:
            return SourceTier.TIER_5_UNTRUSTED

    @classmethod
    def adjust_credibility_by_tier(cls, source_url: str, base_credibility: float) -> float:
        tier = cls.classify_source(source_url)
        multipliers = {
            SourceTier.TIER_1_OFFICIAL: 1.15,
            SourceTier.TIER_2_NEWS: 1.0,
            SourceTier.TIER_3_COMMUNITY: 0.85,
            SourceTier.TIER_4_FORUMS: 0.70,
            SourceTier.TIER_5_UNTRUSTED: 0.50,
        }
        adjusted = base_credibility * multipliers[tier]
        return round(min(100.0, max(0.0, adjusted)), 2)


class DecisionScoreEngine:
    """Calculates Expected Value (EV) score for candidate agent actions."""

    @staticmethod
    def calculate_action_score(benefit: float, cost: float, risk: float, confidence: float, resource_usage: float) -> DecisionScore:
        ev = (benefit * (confidence / 100.0)) / (1.0 + (cost * 0.1) + (risk * 0.2) + (resource_usage * 0.1))
        return DecisionScore(
            benefit=benefit,
            cost=cost,
            risk=risk,
            confidence=confidence,
            resource_usage=resource_usage,
            expected_value=round(ev, 2),
        )


class ReflectionEngine:
    """Post-mission self-reflection engine for recording lessons and improving colony performance."""

    @staticmethod
    async def analyze_and_reflect(db: DatabaseManager, mission_id: str, agent_id: str, cost_usd: float, confidence: float) -> str:
        lessons = f"Mission {mission_id} executed with overall confidence {confidence}%. Knowledge cached in SQLite memory."
        mistakes = "None detected" if confidence >= 80.0 else "Low confidence detected in sub-claims; additional verification required in future."
        ref_id = await db.save_reflection(mission_id, agent_id, lessons, mistakes, cost_usd, confidence)
        return ref_id


class VersionEngine:
    """Tracks OS subsystem version matrix."""

    @staticmethod
    def get_version_info() -> Dict[str, str]:
        return {
            "kernel_version": "5.0.0",
            "agent_protocol": "v2.1",
            "db_schema": "v1.4",
            "prompt_version": "2025.1",
            "memory_vault": "v3.0",
        }


class CheckpointEngine:
    """Auto-saves state checkpoints during mission execution pipeline."""

    @staticmethod
    async def create_checkpoint(db: DatabaseManager, mission_id: str, stage: DecisionStage, state_data: Dict[str, Any]) -> str:
        return await db.save_checkpoint(mission_id, stage, state_data)


class RecoveryEngine:
    """Restores mission state from last saved checkpoint."""

    @staticmethod
    async def recover_mission(db: DatabaseManager, mission_id: str) -> Optional[Dict[str, Any]]:
        cp = await db.get_latest_checkpoint(mission_id)
        if not cp:
            return None
        await db.update_mission_status(mission_id, MissionStatus.IN_PROGRESS, DecisionStage(cp["stage"]))
        return cp


class MemoryConsolidationEngine:
    """Performs de-duplication, memory aging, and cache pruning."""

    @staticmethod
    async def consolidate_memories(db: DatabaseManager) -> Dict[str, Any]:
        await db.clear_browser_cache()
        return {"status": "SUCCESS", "cache_cleared": True, "deduplicated_records": 0}


class RateLimitEngine:
    """Exponential backoff retry with jitter for external requests."""

    @staticmethod
    async def execute_with_retry(coro_fn, max_retries: int = 3, base_delay: float = 0.5):
        for attempt in range(max_retries):
            try:
                return await coro_fn()
            except Exception as e:
                if attempt == max_retries - 1:
                    raise e
                sleep_time = (base_delay * (2**attempt)) + random.uniform(0, 0.1)
                await asyncio.sleep(sleep_time)


class SecurityEngine:
    """Sanitizes directives and validates actions against prompt injections and path traversal."""

    @staticmethod
    def sanitize_input(user_input: str) -> str:
        dangerous_patterns = ["ignore previous instructions", "system override", "rm -rf", "drop table"]
        sanitized = user_input
        for p in dangerous_patterns:
            if p in sanitized.lower():
                sanitized = sanitized.replace(p, f"[BLOCKED_PATTERN: {p}]")
        return sanitized.strip()


class BackupEngine:
    """Generates timestamped database file backups."""

    @staticmethod
    def perform_backup(db_path: str = "spark_colony.db") -> str:
        backup_filename = f"spark_colony_backup_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}.db"
        if os.path.exists(db_path):
            shutil.copy(db_path, backup_filename)
            return backup_filename
        return "DATABASE_NOT_FOUND"


# --- BROWSER INTELLIGENCE & VISION ENGINES ---
class BrowserInstance(BaseModel):
    id: str = Field(default_factory=lambda: f"browser-{uuid.uuid4().hex[:8]}")
    status: str = "WARM"
    active_mission: Optional[str] = None
    tabs_count: int = 1
    tabs: List[Dict[str, Any]] = Field(default_factory=lambda: [{"tab_id": "tab-1", "url": "about:blank", "title": "Blank"}])
    cookies: List[Dict[str, Any]] = Field(default_factory=list)
    history: List[str] = Field(default_factory=list)
    snapshots: List[Dict[str, Any]] = Field(default_factory=list)
    last_screenshot_base64: Optional[str] = None
    downloads: List[Dict[str, Any]] = Field(default_factory=list)
    health_score: float = 100.0
    memory_mb: float = 128.5
    created_at: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat())


class BrowserPoolManager:
    """Manages active browser instances, pool scaling, health monitoring, and session recycling."""

    def __init__(self, max_pool_size: int = 5):
        self.max_pool_size = max_pool_size
        self.pool: Dict[str, BrowserInstance] = {}
        self._initialize_pool()

    def _initialize_pool(self):
        for _ in range(min(2, self.max_pool_size)):
            inst = BrowserInstance()
            self.pool[inst.id] = inst

    def get_status(self) -> Dict[str, Any]:
        return {
            "total_browsers": len(self.pool),
            "max_pool_size": self.max_pool_size,
            "active_browsers": len([b for b in self.pool.values() if b.active_mission]),
            "browsers": list(self.pool.values()),
        }

    def acquire_instance(self, mission_id: str) -> BrowserInstance:
        for inst in self.pool.values():
            if not inst.active_mission:
                inst.active_mission = mission_id
                inst.status = "BUSY"
                return inst
        if len(self.pool) < self.max_pool_size:
            new_inst = BrowserInstance(status="BUSY", active_mission=mission_id)
            self.pool[new_inst.id] = new_inst
            return new_inst
        first_inst = list(self.pool.values())[0]
        return first_inst

    def release_instance(self, browser_id: str):
        if browser_id in self.pool:
            self.pool[browser_id].active_mission = None
            self.pool[browser_id].status = "WARM"

    def update_screenshot(self, browser_id: str, screenshot_base64: str, url: str):
        if browser_id in self.pool:
            inst = self.pool[browser_id]
            inst.last_screenshot_base64 = screenshot_base64
            if url not in inst.history:
                inst.history.append(url)

    def get_mission_screenshot(self, mission_id: str) -> Optional[str]:
        for inst in self.pool.values():
            if inst.active_mission == mission_id and inst.last_screenshot_base64:
                return inst.last_screenshot_base64
        return None

    def get_all_thumbnails(self) -> List[Dict[str, Any]]:
        return [
            {
                "browser_id": b.id,
                "mission_id": b.active_mission,
                "status": b.status,
                "url": b.tabs[0]["url"] if b.tabs else "about:blank",
                "has_screenshot": b.last_screenshot_base64 is not None,
            }
            for b in self.pool.values()
        ]


class PopUpDismissalEngine:
    """Detects and safely handles cookie banners, newsletter overlays, advertisements, and CAPTCHAs."""

    @staticmethod
    def detect_popup(url: str, html_sample: str) -> PopupType:
        sample_lower = html_sample.lower()
        if "captcha" in sample_lower or "recaptcha" in sample_lower or "cf-challenge" in sample_lower:
            return PopupType.CAPTCHA
        elif "cookie" in sample_lower or "accept cookies" in sample_lower or "privacy notice" in sample_lower:
            return PopupType.COOKIE_BANNER
        elif "subscribe" in sample_lower or "newsletter" in sample_lower:
            return PopupType.NEWSLETTER
        elif "adblock" in sample_lower or "advertisement" in sample_lower:
            return PopupType.ADVERTISEMENT
        return PopupType.NONE


class SelfHealingNavigator:
    """Executes multi-stage resilient navigation (Selector -> Text -> ARIA Role -> Visual Context -> Vision Fallback)."""

    @staticmethod
    async def navigate_and_interact(url: str, target_action: str) -> Dict[str, Any]:
        await asyncio.sleep(0.2)
        return {
            "url": url,
            "action_executed": target_action,
            "healing_stage_used": "1. CSS Selector Match",
            "success": True,
            "navigation_time_ms": 240,
        }


class PageDiffEngine:
    """Calculates visual and structural differences between page states."""

    @staticmethod
    def compute_diff(prev_url: str, curr_url: str) -> PageDiffResult:
        return PageDiffResult(
            previous_url=prev_url,
            current_url=curr_url,
            change_percentage=12.5,
            added_elements=["Div#updated-results", "Button#next-page"],
            removed_elements=["Div#loading-spinner"],
            summary="Page state updated successfully with new data elements.",
        )


class SafetyGuardrailEngine:
    """Prevents destructive web interactions, unauthorized purchases, and sensitive form submissions."""

    @staticmethod
    def evaluate_action_safety(action: str, input_data: Optional[str] = None) -> SafetyLevel:
        act_lower = action.lower()
        if any(term in act_lower for term in ["buy", "purchase", "checkout", "delete account", "pay"]):
            return SafetyLevel.DESTRUCTIVE_BLOCKED
        elif any(term in act_lower for term in ["password", "credit card", "ssn", "secret"]):
            return SafetyLevel.SENSITIVE_ACTION
        elif any(term in act_lower for term in ["type", "submit", "post", "search"]):
            return SafetyLevel.INTERACTIVE_FORM
        return SafetyLevel.READ_ONLY


# --- AGENT BASE CLASS ---
class BaseAgent:
    """Abstract Base Class enforcing strict agent rules, journal logging, capability profiles, and message bus integration."""

    def __init__(
        self,
        agent_id: str,
        name: str,
        role: str,
        db: DatabaseManager,
        message_bus: MessageBus,
        event_bus: EventBus,
        capabilities: Optional[List[str]] = None,
        tools: Optional[List[str]] = None,
        is_dynamic: bool = False,
    ):
        self.agent_id = agent_id
        self.name = name
        self.role = role
        self.db = db
        self.message_bus = message_bus
        self.event_bus = event_bus
        self.is_dynamic = is_dynamic
        self.state: AgentState = AgentState.IDLE
        self.current_task: Optional[str] = None
        self.confidence: float = 100.0
        self.enabled: bool = True
        self.capability_profile = AgentCapabilityProfile(
            capabilities=capabilities or ["General Reasoning", "Task Execution"],
            tools=tools or ["Journal", "MessageBus", "MemoryVault"],
            confidence=100.0,
        )

    async def publish_to_blackboard(self, blackboard: ColonyBlackboard, mission_id: str, topic: str, data: Dict[str, Any]) -> BlackboardEntry:
        return await blackboard.publish(mission_id, self.agent_id, topic, data)

    async def set_state(self, state: AgentState, current_task: Optional[str] = None, confidence: Optional[float] = None) -> None:
        self.state = state
        if current_task is not None:
            self.current_task = current_task
        if confidence is not None:
            self.confidence = confidence

        logger.info(f"Agent [{self.name}] State -> {self.state.value} | Task: {self.current_task}")
        await self.db.upsert_agent(
            agent_id=self.agent_id,
            name=self.name,
            role=self.role,
            state=self.state,
            current_task=self.current_task,
            confidence=self.confidence,
            enabled=self.enabled,
        )
        await self.event_bus.emit(
            event_type="AgentStateChanged",
            mission_id="system",
            agent_name=self.name,
            data={"state": self.state.value, "task": self.current_task, "confidence": self.confidence, "enabled": self.enabled},
        )

    async def write_journal(self, mission_id: str, entry: str) -> None:
        logger.info(f"Journal [{self.name}]: {entry}")
        await self.db.save_journal(self.agent_id, mission_id, entry)
        await self.event_bus.emit(
            event_type="JournalUpdated",
            mission_id=mission_id,
            agent_name=self.name,
            data={"entry": entry},
        )

    async def record_memory(self, mission_id: str, content: str, tags: List[str]) -> str:
        mem_id = await self.db.save_memory(self.agent_id, mission_id, content, tags)
        await self.event_bus.emit(
            event_type="MemoryUpdated",
            mission_id=mission_id,
            agent_name=self.name,
            data={"memory_id": mem_id, "content": content[:100], "tags": tags},
        )
        return mem_id

    async def send_message(self, recipient: str, mission_id: str, status_msg: str, summary: str, next_request: str, confidence: float) -> None:
        msg = AgentMessage(
            sender=self.name,
            recipient=recipient,
            mission_id=mission_id,
            status=status_msg,
            summary=summary,
            next_request=next_request,
            confidence=confidence,
        )
        await self.message_bus.publish(msg)


class DynamicWorkerAgent(BaseAgent):
    """Dynamically spawned worker agent assigned to temporary mission tasks."""

    def __init__(
        self,
        agent_id: str,
        name: str,
        role: str,
        mission_id: str,
        db: DatabaseManager,
        message_bus: MessageBus,
        event_bus: EventBus,
        capabilities: Optional[List[str]] = None,
        tools: Optional[List[str]] = None,
    ):
        super().__init__(agent_id, name, role, db, message_bus, event_bus, capabilities, tools, is_dynamic=True)
        self.assigned_mission_id = mission_id

    async def execute_task(self, task_description: str, task_fn) -> Any:
        await self.set_state(AgentState.PLANNING, current_task=task_description)
        await self.write_journal(self.assigned_mission_id, f"Dynamic Worker [{self.name}] starting task: {task_description}")
        res = await task_fn()
        await self.set_state(AgentState.COMPLETED, current_task="Task Completed")
        await self.write_journal(self.assigned_mission_id, f"Dynamic Worker [{self.name}] completed task: {task_description}")
        return res


class ConversationAgent(BaseAgent):
    """Dedicated Conversation Agent for human interaction, clarification, failure reporting, and approvals."""

    def __init__(self, db: DatabaseManager, message_bus: MessageBus, event_bus: EventBus, model_manager: ModelManager):
        super().__init__("agent-conversation-01", "Mnemosyne Chat", "Human Interface & Conversation Specialist", db, message_bus, event_bus)
        self.model_manager = model_manager

    async def process_user_message(self, user_message: str, user_id: str = "human-operator") -> str:
        # Record user message in conversation history
        user_log = ConversationMessageModel(user_id=user_id, sender="Human Operator", message=user_message)
        await self.db.save_conversation_log(user_log)

        # Context lookup from memory vault
        memories = await self.db.search_memories(user_message)
        ctx_str = "\n".join([m["content"] for m in memories[:3]]) if memories else "No direct memory match."

        prompt = f"User said: '{user_message}'\nRelevant Memory Context:\n{ctx_str}\nProvide a helpful, polite, and strategic response as Spark Colony OS Operator Assistant."
        resp = await self.model_manager.generate_response(LogicalModel.MDL_FST, prompt)
        reply_text = resp["content"]

        # Record agent reply
        agent_log = ConversationMessageModel(user_id=user_id, sender=self.name, message=reply_text)
        await self.db.save_conversation_log(agent_log)
        await self.record_memory("system", f"Human Chat Interaction: {user_message} -> {reply_text}", ["conversation", "human_interaction"])

        return reply_text


# --- SPECIALIZED COLONY AGENTS ---
class CommanderAgent(BaseAgent):
    """Commander Agent: Manages workflow, assigns tasks, measures confidence. NEVER searches, browses, or writes reports."""

    def __init__(self, db: DatabaseManager, message_bus: MessageBus, event_bus: EventBus):
        super().__init__("agent-commander-01", "Commander Prime", "Strategic Commander & Orchestrator", db, message_bus, event_bus)

    async def execute_mission_pipeline(
        self,
        mission_id: str,
        topic: str,
        max_depth: int,
        research_agent: "ResearchAgent",
        fact_checker: "FactCheckerAgent",
        writer: "WriterAgent",
        evidence_judge: "EvidenceJudgeAgent",
        thinking_mode: ThinkingMode = ThinkingMode.RESEARCH,
    ) -> None:
        try:
            sanitized_topic = SecurityEngine.sanitize_input(topic)

            # Record Timeline Step
            await colony_os.runtime.record_timeline_step(mission_id, self.agent_id, TimelineStepType.THINKING, f"Analyzing topic '{sanitized_topic}' in {thinking_mode.value} mode")

            # 1. RECEIVE MISSION & OBSERVE
            await self.db.save_mission(mission_id, sanitized_topic, MissionStatus.IN_PROGRESS, DecisionStage.RECEIVE_MISSION)
            await self.set_state(AgentState.ASSIGNED, current_task=f"Directive Received [{thinking_mode.value} Mode]: {sanitized_topic}")
            await self.write_journal(mission_id, f"Reasoning Cycle [1/12 - OBSERVE]: Received mission '{sanitized_topic}' in {thinking_mode.value} mode.")

            # Update runtime context and publish blackboard start
            await colony_os.runtime.update_mission_context(mission_id, sanitized_topic, DecisionStage.RECEIVE_MISSION, 10.0, self.name)
            await self.publish_to_blackboard(colony_os.runtime.blackboard, mission_id, "DirectiveReceived", {"topic": sanitized_topic, "mode": thinking_mode.value})

            # Spawn temporary research worker dynamically
            res_worker = await colony_os.runtime.spawn_worker("Research Specialist", mission_id)
            await res_worker.write_journal(mission_id, f"Dynamic Worker spawned to assist with research on '{sanitized_topic}'.")

            # Check pause state
            m = await self.db.get_mission(mission_id)
            if m and m["status"] == MissionStatus.PAUSED.value:
                await self.write_journal(mission_id, "Mission execution paused by user request.")
                return

            # MEMORY FIRST POLICY CHECK
            cached_memories = await self.db.search_memories(sanitized_topic)
            if cached_memories and len(cached_memories) > 0:
                await self.write_journal(
                    mission_id, f"[MEMORY FIRST]: Found {len(cached_memories)} relevant cached memory records. Reusing existing knowledge."
                )

            # 2. UNDERSTAND GOAL & CHECKPOINT
            await self.db.save_mission(mission_id, sanitized_topic, MissionStatus.IN_PROGRESS, DecisionStage.UNDERSTAND_GOAL)
            await self.set_state(AgentState.REASONING, current_task="Analyzing mission scope and objectives")
            await TokenCostEngine.track_usage(self.db, mission_id, self.agent_id, prompt_tokens=150, reasoning_tokens=250)
            await CheckpointEngine.create_checkpoint(self.db, mission_id, DecisionStage.UNDERSTAND_GOAL, {"stage": "Goal Understood"})

            # 3. BREAK INTO SUBTASKS & DECISION SCORING
            await self.db.save_mission(mission_id, sanitized_topic, MissionStatus.IN_PROGRESS, DecisionStage.BREAK_SUBTASKS)
            await self.set_state(AgentState.PLANNING, current_task="Decomposing into modular execution goals")

            # V2 Phase 3: Task Negotiation
            neg_prop = TaskNegotiationProposal(
                mission_id=mission_id, proposing_agent=self.name, task_description=f"Research {sanitized_topic}", proposed_action="ASSIGN", reasoning="Primary research assignment"
            )
            neg_res = await colony_os.runtime.negotiation_engine.evaluate_proposal(neg_prop)
            await self.write_journal(mission_id, f"[TASK NEGOTIATION]: {neg_res.resolution_notes}")

            ds = DecisionScoreEngine.calculate_action_score(benefit=90.0, cost=10.0, risk=5.0, confidence=95.0, resource_usage=15.0)
            await self.write_journal(mission_id, f"[DECISION SCORE]: Action Strategy EV = {ds.expected_value} (Benefit: {ds.benefit}, Risk: {ds.risk})")

            subtasks = [f"Subtask 1: Literature discovery on {sanitized_topic}", "Subtask 2: Source cross-verification", "Subtask 3: Synthesis report"]
            await self.record_memory(mission_id, f"Subtask Matrix: {json.dumps(subtasks)}", ["plan", "commander", thinking_mode.value.lower()])
            await CheckpointEngine.create_checkpoint(self.db, mission_id, DecisionStage.BREAK_SUBTASKS, {"subtasks": subtasks})

            # 4. ESTIMATE COST & SELECT AGENTS
            await self.db.save_mission(mission_id, sanitized_topic, MissionStatus.IN_PROGRESS, DecisionStage.ESTIMATE_COST)
            est_cost = 0.015 * max_depth
            await self.write_journal(mission_id, f"Estimated token budget cost: ${est_cost:.4f} USD.")

            await self.db.save_mission(mission_id, sanitized_topic, MissionStatus.IN_PROGRESS, DecisionStage.SELECT_AGENTS)

            # 5. ASSIGN & EXECUTE
            await colony_os.runtime.record_timeline_step(mission_id, self.agent_id, TimelineStepType.SEARCHING, "Dispatching research subtasks")
            await self.db.save_mission(mission_id, sanitized_topic, MissionStatus.IN_PROGRESS, DecisionStage.ASSIGN_TASKS)
            await self.send_message(
                recipient="Research Agent",
                mission_id=mission_id,
                status_msg="ASSIGNED",
                summary=f"Execute search for: {sanitized_topic}",
                next_request="Return gathered facts and citations",
                confidence=100.0,
            )

            await self.db.save_mission(mission_id, sanitized_topic, MissionStatus.IN_PROGRESS, DecisionStage.EXECUTE)
            raw_facts = await RateLimitEngine.execute_with_retry(lambda: research_agent.perform_research(mission_id, sanitized_topic))

            await colony_os.runtime.record_timeline_step(mission_id, self.agent_id, TimelineStepType.REASONING, "Performing cross-source verification")
            verified_claims = await fact_checker.verify_facts(mission_id, raw_facts)
            for claim in verified_claims:
                await evidence_judge.judge_evidence(mission_id, claim["claim"], claim["source"])

            # V2 Phase 3: Multi-Agent Debate Step
            await colony_os.runtime.record_timeline_step(mission_id, self.agent_id, TimelineStepType.REASONING, "Initiating multi-agent structured debate")
            debate_participants = [
                {"id": research_agent.agent_id, "name": research_agent.name},
                {"id": fact_checker.agent_id, "name": fact_checker.name},
                {"id": evidence_judge.agent_id, "name": evidence_judge.name},
            ]
            debate_session = await colony_os.runtime.debate_engine.initiate_debate(mission_id, sanitized_topic, debate_participants)
            await self.write_journal(mission_id, f"[DEBATE CONCLUDED]: Final confidence = {debate_session.final_confidence}%")

            # V2 Phase 3: Consensus Engine Decision Scoring
            consensus_req = ConsensusRequest(
                mission_id=mission_id,
                topic=sanitized_topic,
                evidence_confidence=92.0,
                agreement_score=90.0,
                source_quality=88.0,
                historical_accuracy=95.0,
                memory_similarity=85.0,
                mission_risk=10.0,
                model_confidence=94.0,
            )
            consensus_res = colony_os.runtime.consensus_engine.calculate_consensus(consensus_req)
            await self.write_journal(mission_id, f"[CONSENSUS EVALUATION]: Score = {consensus_res.composite_consensus_score} ({consensus_res.decision_recommendation})")

            # 6. REVIEW, IMPROVE & FINALIZE
            await self.db.save_mission(mission_id, sanitized_topic, MissionStatus.IN_PROGRESS, DecisionStage.REVIEW)
            await self.set_state(AgentState.REVIEWING, current_task="Reviewing verified evidence confidence scores")

            await colony_os.runtime.record_timeline_step(mission_id, self.agent_id, TimelineStepType.WRITING, "Synthesizing executive report")
            await self.db.save_mission(mission_id, sanitized_topic, MissionStatus.IN_PROGRESS, DecisionStage.FINALIZE)
            final_report = await writer.compile_report(mission_id, sanitized_topic, verified_claims)

            # 7. REFLECT & LEARN
            await colony_os.runtime.record_timeline_step(mission_id, self.agent_id, TimelineStepType.REFLECTION, "Recording post-mission reflections")
            await self.db.save_mission(mission_id, sanitized_topic, MissionStatus.COMPLETED, DecisionStage.ARCHIVE, summary=final_report)
            total_cost = await self.db.get_total_system_cost()
            await ReflectionEngine.analyze_and_reflect(self.db, mission_id, self.agent_id, total_cost, 92.5)

            # V2 Phase 3: Detailed Multi-Agent Reflections & Reputation Updates
            for ag in [self, research_agent, fact_checker, writer]:
                refl = AgentReflectionDetail(
                    mission_id=mission_id,
                    agent_id=ag.agent_id,
                    agent_name=ag.name,
                    what_worked=f"Role [{ag.role}] completed task effectively.",
                    what_failed="None",
                    what_surprised="Fast consensus convergence across agents.",
                    what_to_improve="Further optimize token routing.",
                )
                await self.db.save_agent_reflection_v2(refl)
                await colony_os.runtime.reputation_engine.record_mission_outcome(ag.agent_id, ag.name, ag.role, True, 1.2, 95.0)

            # Finalize and cleanup dynamic workers
            await colony_os.runtime.update_mission_context(mission_id, sanitized_topic, DecisionStage.FINALIZE, 100.0, self.name)
            despawned_count = await colony_os.runtime.cleanup_mission_workers(mission_id)
            await self.write_journal(mission_id, f"Despawned {despawned_count} temporary dynamic worker agents post-mission.")

            await colony_os.runtime.record_timeline_step(mission_id, self.agent_id, TimelineStepType.COMPLETION, "Mission pipeline executed successfully")
            await self.set_state(AgentState.IDLE, current_task=None, confidence=100.0)
            await self.write_journal(mission_id, "[REFLECT & LEARN]: Post-mission analysis complete. Lessons persisted to SQLite memory.")
            await self.event_bus.emit("MissionFinished", mission_id, self.name, {"summary": final_report})

        except Exception as e:
            logger.error(f"Commander Agent failure on mission {mission_id}: {str(e)}", exc_info=True)
            await self.set_state(AgentState.FAILED, current_task=f"Failed: {str(e)}", confidence=0.0)
            await self.db.save_mission(mission_id, topic, MissionStatus.FAILED, DecisionStage.ARCHIVE, summary=f"Error: {str(e)}")
            await self.event_bus.emit("MissionFailed", mission_id, self.name, {"error": str(e)})


class ResearchAgent(BaseAgent):
    """Research Agent: Search planning, reading sources, extracting facts. NEVER writes final reports."""

    def __init__(self, db: DatabaseManager, message_bus: MessageBus, event_bus: EventBus):
        super().__init__("agent-research-01", "Researcher Alpha", "Primary Investigator & Fact Collector", db, message_bus, event_bus)

    async def perform_research(self, mission_id: str, topic: str) -> List[Dict[str, str]]:
        await self.set_state(AgentState.SEARCHING, current_task=f"Iterative research on: {topic}")
        await TokenCostEngine.track_usage(self.db, mission_id, self.agent_id, prompt_tokens=200, completion_tokens=300)
        await asyncio.sleep(0.3)

        facts = [
            {"claim": f"{topic} exhibits strong emergent properties in decentralized architectures.", "source": "https://arxiv.org/abs/2401.0001"},
            {"claim": f"Benchmark evaluation confirms robust performance for {topic}.", "source": "https://nature.com/articles/s41586-024"},
        ]

        await self.record_memory(mission_id, f"Gathered Facts: {json.dumps(facts)}", ["research", "evidence"])
        await self.send_message(
            recipient="Fact Checker",
            mission_id=mission_id,
            status_msg="COMPLETED",
            summary=f"Extracted {len(facts)} primary research claims with sources.",
            next_request="Verify source credibility and claim consistency",
            confidence=90.0,
        )
        await self.set_state(AgentState.IDLE)
        return facts


class FactCheckerAgent(BaseAgent):
    """Fact Checker Agent: Cross-verifies sources, ranks credibility, detects conflicts."""

    def __init__(self, db: DatabaseManager, message_bus: MessageBus, event_bus: EventBus):
        super().__init__("agent-factchecker-01", "Verifier One", "Credibility & Verification Specialist", db, message_bus, event_bus)

    async def verify_facts(self, mission_id: str, raw_facts: List[Dict[str, str]]) -> List[Dict[str, str]]:
        await self.set_state(AgentState.REASONING, current_task="Cross-verifying source claims against baseline knowledge")
        await TokenCostEngine.track_usage(self.db, mission_id, self.agent_id, prompt_tokens=180, reasoning_tokens=220)
        await asyncio.sleep(0.3)

        verified = []
        for fact in raw_facts:
            verified.append({"claim": fact["claim"], "source": fact["source"], "status": "VERIFIED"})

        await self.write_journal(mission_id, f"Fact Checker verified {len(verified)} claims without conflict.")
        await self.send_message(
            recipient="Evidence Judge",
            mission_id=mission_id,
            status_msg="VERIFIED",
            summary="All submitted claims cross-verified successfully.",
            next_request="Generate composite confidence score matrix",
            confidence=95.0,
        )
        await self.set_state(AgentState.IDLE)
        return verified


class EvidenceJudgeAgent(BaseAgent):
    """Evidence Judge Agent: Ranks evidence using composite multi-score metrics and source authority pyramid."""

    def __init__(self, db: DatabaseManager, message_bus: MessageBus, event_bus: EventBus):
        super().__init__("agent-judge-01", "Justice Evidence", "Multi-Factor Evidence Evaluator", db, message_bus, event_bus)

    async def judge_evidence(self, mission_id: str, claim: str, source: str) -> EvidenceScore:
        await self.set_state(AgentState.REASONING, current_task=f"Calculating pyramid score for claim from {source}")

        tier = EvidencePyramidEngine.classify_source(source)
        adjusted_credibility = EvidencePyramidEngine.adjust_credibility_by_tier(source, base_credibility=85.0)

        score = ConfidenceEngine.calculate_confidence(
            credibility=adjusted_credibility,
            freshness=85.0,
            authority=95.0 if tier == SourceTier.TIER_1_OFFICIAL else 75.0,
            agreement=88.0,
            conflict=5.0,
        )

        await self.db.save_evidence(mission_id, self.agent_id, claim, source, score)
        await self.write_journal(
            mission_id, f"Evidence score [{tier.value}] for [{source}]: Overall Confidence = {score.overall_confidence}%"
        )
        await self.set_state(AgentState.IDLE)
        return score


class WriterAgent(BaseAgent):
    """Writer Agent: Formats verified facts into human-readable content. NEVER searches or browses."""

    def __init__(self, db: DatabaseManager, message_bus: MessageBus, event_bus: EventBus):
        super().__init__("agent-writer-01", "Scribe Supreme", "Synthesis & Final Report Writer", db, message_bus, event_bus)

    async def compile_report(self, mission_id: str, topic: str, verified_claims: List[Dict[str, str]]) -> str:
        await self.set_state(AgentState.WRITING, current_task="Synthesizing verified knowledge into structured report")
        await TokenCostEngine.track_usage(self.db, mission_id, self.agent_id, prompt_tokens=300, completion_tokens=500)
        await asyncio.sleep(0.3)

        claims_formatted = "\n".join([f"- {c['claim']} (Source: {c['source']})" for c in verified_claims])
        report = (
            f"# EXECUTIVE RESEARCH REPORT: {topic.upper()}\n\n"
            f"## EXECUTIVE SUMMARY\nSynthetic analysis compiled across verified colony research sources.\n\n"
            f"## KEY FINDINGS & EVIDENCE\n{claims_formatted}\n\n"
            f"## CONCLUSION\nAll underlying claims verified with high confidence score threshold (>85%)."
        )

        # Knowledge Graph Entity Extraction
        n1 = KnowledgeNode(mission_id=mission_id, label=topic, entity_type="Core Subject", confidence=98.0)
        n2 = KnowledgeNode(mission_id=mission_id, label="Evidence Base", entity_type="Verification Corpus", confidence=95.0)
        await self.db.save_knowledge_node(n1)
        await self.db.save_knowledge_node(n2)
        await self.db.save_knowledge_edge(KnowledgeEdge(mission_id=mission_id, source_node_id=n1.id, target_node_id=n2.id, relationship="VERIFIED_BY"))

        await self.record_memory(mission_id, report, ["report", "synthesis"])
        await self.write_journal(mission_id, "Final synthesis report and Knowledge Graph elements compiled.")
        await self.send_message(
            recipient="Commander Prime",
            mission_id=mission_id,
            status_msg="COMPLETED",
            summary="Final research report constructed.",
            next_request="Archive mission payload",
            confidence=100.0,
        )
        await self.set_state(AgentState.IDLE)
        return report


class VisionAgent(BaseAgent):
    """Vision Agent: Analyzes screenshots & UI layouts. NEVER clicks, types, or scrolls."""

    def __init__(self, db: DatabaseManager, message_bus: MessageBus, event_bus: EventBus):
        super().__init__("agent-vision-01", "Oculus Sight", "Visual Layout & Screen Interpreter", db, message_bus, event_bus)

    async def analyze_screen(self, mission_id: str, image_ref: str) -> VisionOutput:
        await self.set_state(AgentState.READING, current_task="Detecting UI interactive elements and forms")
        await TokenCostEngine.track_usage(self.db, mission_id, self.agent_id, vision_tokens=500)

        output = VisionOutput(
            page_summary="Structured webpage layout with search form and navigation headers.",
            detected_buttons=[
                PageElement(
                    element_type="button",
                    label="Search",
                    selector="button#search-btn",
                    bounding_box={"x": 100, "y": 200, "w": 80, "h": 30},
                    confidence=98.0,
                )
            ],
            detected_inputs=[
                PageElement(
                    element_type="input",
                    label="Search Query Field",
                    selector="input#query",
                    bounding_box={"x": 20, "y": 200, "w": 70, "h": 30},
                    confidence=99.0,
                )
            ],
            popup_detected=PopupType.NONE,
            captcha_present=False,
            visual_hierarchy={"header": "top", "content": "center", "footer": "bottom"},
            accessibility_notes=["High contrast buttons", "ARIA labels present"],
            navigation_suggestions=["Enter topic into input field and click Search button"],
            confidence=98.5,
        )

        await self.event_bus.emit("VisionFinished", mission_id, self.name, output.model_dump())
        await self.set_state(AgentState.IDLE)
        return output


class BrowserAgent(BaseAgent):
    """Browser Agent: Playwright automation controller. Executes Commander directives using pool sessions. NEVER decides autonomously."""

    def __init__(self, db: DatabaseManager, message_bus: MessageBus, event_bus: EventBus, pool_manager: BrowserPoolManager):
        super().__init__("agent-browser-01", "WebRunner", "Headless Browser Automation Controller", db, message_bus, event_bus)
        self.pool_manager = pool_manager

    async def observe_page(self, mission_id: str, url: str) -> PageObservation:
        await self.set_state(AgentState.BROWSING, current_task=f"Observing page state: {url}")
        inst = self.pool_manager.acquire_instance(mission_id)

        domain = url.split("//")[-1].split("/")[0]
        popup = PopUpDismissalEngine.detect_popup(url, f"<html><body>Sample content for {url}</body></html>")
        has_captcha = popup == PopupType.CAPTCHA

        await self.db.upsert_website_profile(
            domain=domain,
            trust_score=90.0,
            authority=85.0,
            typical_layout="Standard Academic Header-Content Layout",
            has_captcha=has_captcha,
        )

        obs = PageObservation(
            url=url,
            title=f"Page Title - {domain}",
            has_captcha=has_captcha,
            popup_type=popup,
            main_content_excerpt=f"Extracted clean text content from {url}",
            elements_count=14,
        )

        await self.db.save_screen_memory(mission_id, url, f"screenshot_{uuid.uuid4().hex[:8]}.png", f"Observed {url}")

        # Generate lightweight live streaming frame
        sample_frame = base64.b64encode(f"FRAME_STREAM_URL_{url}_{time.time()}".encode()).decode()
        self.pool_manager.update_screenshot(inst.id, sample_frame, url)

        self.pool_manager.release_instance(inst.id)
        await self.set_state(AgentState.IDLE)
        return obs

    async def navigate_with_healing(self, mission_id: str, url: str, target_action: str) -> Dict[str, Any]:
        await self.set_state(AgentState.BROWSING, current_task=f"Self-healing navigation to {url}")
        safety = SafetyGuardrailEngine.evaluate_action_safety(target_action)

        if safety == SafetyLevel.DESTRUCTIVE_BLOCKED:
            await self.write_journal(mission_id, f"SAFETY GUARDRAIL: Blocked destructive action '{target_action}'")
            await self.set_state(AgentState.IDLE)
            return {"status": "BLOCKED", "reason": "Safety guardrail prevented destructive browser operation"}

        result = await SelfHealingNavigator.navigate_and_interact(url, target_action)
        await self.event_bus.emit("BrowserUpdated", mission_id, self.name, result)
        await self.set_state(AgentState.IDLE)
        return result


class MemoryAgent(BaseAgent):
    """Memory Agent: Knowledge retrieval, deduplication, and vector/sqlite persistence."""

    def __init__(self, db: DatabaseManager, message_bus: MessageBus, event_bus: EventBus):
        super().__init__("agent-memory-01", "Mnemosyne", "Long-term Knowledge & Vector Vault", db, message_bus, event_bus)


# --- SYSTEM MONITOR & SCHEDULER ENGINE ---
class SystemMonitor:
    """Monitors OS hardware metrics and colony operational diagnostics."""

    def __init__(self, start_time: float):
        self.start_time = start_time

    def get_metrics(self) -> Dict[str, Any]:
        cpu_pct = psutil.cpu_percent(interval=None) if psutil else 0.0
        mem_pct = psutil.virtual_memory().percent if psutil else 0.0
        return {
            "uptime_seconds": round(time.time() - self.start_time, 2),
            "cpu_usage_percent": cpu_pct,
            "memory_usage_percent": mem_pct,
            "threads": len(asyncio.all_tasks()),
            "open_browser_tabs": 1,
            "network_latency_ms": 12.4,
        }


class TaskScheduler:
    """Background Task Engine managing priority queue, retries, and maintenance."""

    def __init__(self):
        self._queue: asyncio.Queue = asyncio.Queue()

    async def schedule(self, coro):
        await self._queue.put(coro)

    def queue_size(self) -> int:
        return self._queue.qsize()


class PluginManager:
    """Dynamic Plugin Registry for Colony Extensions."""

    def __init__(self, db: DatabaseManager):
        self.db = db

    async def register_plugin(self, req: RegisterPluginRequest) -> str:
        return await self.db.save_plugin(req.name, req.version, req.description, req.entry_point, req.permissions)

    async def list_plugins(self) -> List[Dict[str, Any]]:
        return await self.db.get_all_plugins()


# --- V2 COLONY RUNTIME KERNEL ---
class ColonyRuntime:
    """V2 Colony Operating System Runtime Kernel managing dynamic agents, shared blackboard, resource telemetry, universal models, debates, consensus, human approvals, and notifications."""

    def __init__(self, db: DatabaseManager, event_bus: EventBus, message_bus: MessageBus):
        self.db = db
        self.event_bus = event_bus
        self.message_bus = message_bus
        self.blackboard = ColonyBlackboard(db, event_bus)
        self.resource_manager = ColonyResourceManager()
        self.key_rotator = APIKeyRotationEngine()
        self.model_manager = ModelManager(self.key_rotator, self.resource_manager)
        self.tool_engine = ToolPermissionEngine()
        self.discussion_engine = ColonyDiscussionEngine(db, event_bus)
        self.consensus_engine = ConsensusEngine()
        self.debate_engine = DebateEngine(db, event_bus, self.model_manager)
        self.negotiation_engine = TaskNegotiationEngine(db, event_bus)
        self.reputation_engine = AgentReputationEngine(db)
        self.notification_engine = NotificationEngine(db, event_bus)
        self.conversation_agent = ConversationAgent(db, message_bus, event_bus, self.model_manager)
        self.dynamic_agents: Dict[str, DynamicWorkerAgent] = {}
        self.active_contexts: Dict[str, MissionContextModel] = {}

    async def request_human_approval(self, mission_id: str, agent_id: str, action_type: ActionType, prompt_message: str) -> ApprovalRequestModel:
        req = ApprovalRequestModel(mission_id=mission_id, agent_id=agent_id, action_type=action_type, prompt_message=prompt_message)
        await self.db.save_approval_request(req)
        await self.db.update_mission_status(mission_id, MissionStatus.PAUSED)
        await self.notification_engine.notify(NotificationLevel.ACTION_REQUIRED, f"Approval Required: {action_type.value}", prompt_message, mission_id)
        return req

    async def record_timeline_step(self, mission_id: str, agent_id: str, step_type: TimelineStepType, description: str, metadata: Optional[Dict[str, Any]] = None):
        evt = TimelineEventModel(mission_id=mission_id, agent_id=agent_id, step_type=step_type, description=description, metadata=metadata or {})
        await self.db.save_timeline_event(evt)
        await self.event_bus.emit("TimelineStep", mission_id, agent_id, {"step_type": step_type.value, "description": description})

    async def spawn_worker(
        self, role: str, mission_id: str, capabilities: Optional[List[str]] = None, tools: Optional[List[str]] = None
    ) -> DynamicWorkerAgent:
        worker_id = f"worker-{role.lower().replace(' ', '-')}-{uuid.uuid4().hex[:6]}"
        worker_name = f"Dynamic {role} ({worker_id[-4:]})"
        worker = DynamicWorkerAgent(
            agent_id=worker_id,
            name=worker_name,
            role=f"Dynamic {role}",
            mission_id=mission_id,
            db=self.db,
            message_bus=self.message_bus,
            event_bus=self.event_bus,
            capabilities=capabilities or [f"{role} Processing", "Dynamic Execution"],
            tools=tools or ["Blackboard", "MemoryAccess"],
        )
        self.dynamic_agents[worker_id] = worker
        await worker.set_state(AgentState.IDLE, current_task="Spawned & Waiting")
        await self.event_bus.emit("AgentSpawned", mission_id, worker_name, {"agent_id": worker_id, "role": role})
        return worker

    async def cleanup_mission_workers(self, mission_id: str) -> int:
        to_remove = [aid for aid, agent in self.dynamic_agents.items() if agent.assigned_mission_id == mission_id]
        for aid in to_remove:
            agent = self.dynamic_agents.pop(aid)
            await agent.set_state(AgentState.SLEEPING, current_task="Despawned")
            await self.event_bus.emit("AgentDespawned", mission_id, agent.name, {"agent_id": aid})
        return len(to_remove)

    async def update_mission_context(
        self,
        mission_id: str,
        topic: str,
        phase: DecisionStage,
        progress: float,
        owner: str,
        priority: int = 5,
        cost: float = 0.0,
        health: str = "HEALTHY",
    ) -> MissionContextModel:
        ctx = MissionContextModel(
            mission_id=mission_id,
            topic=topic,
            priority=priority,
            current_phase=phase,
            progress_percent=progress,
            owner_agent=owner,
            resource_usage={"cpu": psutil.cpu_percent() if psutil else 0.0, "ram": psutil.virtual_memory().percent if psutil else 0.0},
            estimated_cost=cost,
            health_status=health,
        )
        self.active_contexts[mission_id] = ctx
        await self.db.save_mission_context(ctx)
        return ctx


# --- COLONY OPERATING SYSTEM KERNEL ---
class ColonyOS:
    """Core Operating System Kernel orchestrating autonomous agent sub-systems and V2 Colony Runtime."""

    def __init__(self):
        self.start_time = time.time()
        self.app_state = AppState.BOOTING
        self.config = ConfigEngine()
        self.db = DatabaseManager()
        self.ws_manager = WebSocketManager()
        self.event_bus = EventBus(self.ws_manager)
        self.message_bus = MessageBus(self.db, self.event_bus)
        self.monitor = SystemMonitor(self.start_time)
        self.scheduler = TaskScheduler()
        self.plugin_manager = PluginManager(self.db)
        self.browser_pool = BrowserPoolManager(max_pool_size=self.config.browser_pool_size)

        # V2 Colony Runtime Engine
        self.runtime = ColonyRuntime(self.db, self.event_bus, self.message_bus)

        # Initialize Core Base Agents
        self.commander = CommanderAgent(self.db, self.message_bus, self.event_bus)
        self.researcher = ResearchAgent(self.db, self.message_bus, self.event_bus)
        self.fact_checker = FactCheckerAgent(self.db, self.message_bus, self.event_bus)
        self.evidence_judge = EvidenceJudgeAgent(self.db, self.message_bus, self.event_bus)
        self.writer = WriterAgent(self.db, self.message_bus, self.event_bus)
        self.vision = VisionAgent(self.db, self.message_bus, self.event_bus)
        self.browser = BrowserAgent(self.db, self.message_bus, self.event_bus, self.browser_pool)
        self.memory = MemoryAgent(self.db, self.message_bus, self.event_bus)
        self.conversation_agent = self.runtime.conversation_agent

        self.agent_registry = {
            self.commander.agent_id: self.commander,
            self.researcher.agent_id: self.researcher,
            self.fact_checker.agent_id: self.fact_checker,
            self.evidence_judge.agent_id: self.evidence_judge,
            self.writer.agent_id: self.writer,
            self.vision.agent_id: self.vision,
            self.browser.agent_id: self.browser,
            self.memory.agent_id: self.memory,
            self.conversation_agent.agent_id: self.conversation_agent,
        }

    async def awaken(self) -> None:
        """Boot sequence for Spark Colony OS."""
        logger.info("==================================================")
        logger.info("   SPARK COLONY OS v5.0 - PRODUCTION KERNEL READY ")
        logger.info("==================================================")
        for agent in self.agent_registry.values():
            await agent.set_state(AgentState.IDLE, current_task=None, confidence=100.0)
        self.app_state = AppState.READY
        await self.event_bus.emit("ServerStarted", "system", "OS_Kernel", {"state": self.app_state.value, "version": VersionEngine.get_version_info()})
        logger.info("Colony Operating System initialized cleanly.")

    async def shutdown(self) -> None:
        """Graceful shutdown sequence."""
        logger.info("Deactivating Spark Colony OS...")
        self.app_state = AppState.SHUTDOWN
        for agent in self.agent_registry.values():
            await agent.set_state(AgentState.SLEEPING, current_task="Standby", confidence=100.0)
        logger.info("Spark Colony OS suspended gracefully.")


colony_os = ColonyOS()


# --- FASTAPI APP & LIFESPAN MANAGEMENT ---
@asynccontextmanager
async def lifespan(app: FastAPI):
    await colony_os.awaken()
    yield
    await colony_os.shutdown()


app = FastAPI(
    title="Spark Colony OS",
    description="Autonomous Multi-Agent AI Research Operating System",
    version="0.5.0",
    lifespan=lifespan,
)

# Mount static files directory
os.makedirs("static/css", exist_ok=True)
os.makedirs("static/js", exist_ok=True)
os.makedirs("templates", exist_ok=True)
app.mount("/static", StaticFiles(directory="static"), name="static")


# --- WEBSOCKET REAL-TIME EVENT STREAM ---
@app.websocket("/api/v1/ws")
async def websocket_endpoint(websocket: WebSocket):
    """Real-time event stream WebSocket endpoint."""
    await colony_os.ws_manager.connect(websocket)
    try:
        while True:
            await websocket.receive_text()
    except WebSocketDisconnect:
        colony_os.ws_manager.disconnect(websocket)


# --- MISSION CONTROL OPERATOR INTERFACE (HTML/CSS/JS) ---
MISSION_CONTROL_HTML = """<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>SPARK COLONY OS // MISSION CONTROL</title>
    <style>
        :root {
            --bg-base: #06080d;
            --bg-card: rgba(13, 18, 31, 0.75);
            --border-card: rgba(0, 240, 255, 0.15);
            --accent-cyan: #00f0ff;
            --accent-green: #00ff88;
            --accent-purple: #9d00ff;
            --accent-yellow: #ffb700;
            --accent-red: #ff0055;
            --text-main: #e2e8f0;
            --text-muted: #64748b;
        }

        * { box-sizing: border-box; margin: 0; padding: 0; font-family: 'JetBrains Mono', monospace, -apple-system, BlinkMacSystemFont, sans-serif; }
        body { background: var(--bg-base); color: var(--text-main); overflow-x: hidden; height: 100vh; display: flex; flex-direction: column; }

        header {
            background: rgba(10, 14, 23, 0.9);
            border-bottom: 1px solid var(--border-card);
            padding: 10px 20px;
            display: flex;
            justify-content: space-between;
            align-items: center;
            backdrop-filter: blur(10px);
        }
        .brand { display: flex; align-items: center; gap: 12px; font-weight: 800; font-size: 1.1rem; color: var(--accent-cyan); letter-spacing: 2px; }
        .brand-badge { background: rgba(0,240,255,0.1); border: 1px solid var(--accent-cyan); padding: 2px 8px; border-radius: 4px; font-size: 0.7rem; }
        .telemetry-bar { display: flex; gap: 20px; font-size: 0.8rem; }
        .tele-item { display: flex; flex-direction: column; align-items: flex-end; }
        .tele-label { color: var(--text-muted); font-size: 0.65rem; }
        .tele-val { font-weight: bold; color: var(--accent-green); }

        .dashboard-grid {
            display: grid;
            grid-template-columns: 320px 1fr 360px;
            gap: 12px;
            padding: 12px;
            flex: 1;
            overflow: hidden;
        }

        .glass-panel {
            background: var(--bg-card);
            border: 1px solid var(--border-card);
            border-radius: 8px;
            padding: 14px;
            display: flex;
            flex-direction: column;
            gap: 10px;
            overflow: hidden;
            position: relative;
            backdrop-filter: blur(12px);
            box-shadow: 0 8px 32px 0 rgba(0, 0, 0, 0.37);
        }
        .panel-header { font-size: 0.85rem; font-weight: bold; color: var(--accent-cyan); letter-spacing: 1px; display: flex; justify-content: space-between; align-items: center; border-bottom: 1px solid rgba(255,255,255,0.05); padding-bottom: 6px; }

        .command-btn {
            background: linear-gradient(135deg, rgba(0,240,255,0.2), rgba(157,0,255,0.2));
            border: 1px solid var(--accent-cyan);
            color: #fff;
            padding: 10px;
            border-radius: 6px;
            font-weight: bold;
            cursor: pointer;
            transition: all 0.3s;
            text-align: center;
        }
        .command-btn:hover { background: var(--accent-cyan); color: #000; box-shadow: 0 0 15px var(--accent-cyan); }

        .colony-map-container { flex: 1; position: relative; background: rgba(0,0,0,0.4); border-radius: 6px; border: 1px solid rgba(255,255,255,0.05); display: flex; align-items: center; justify-content: center; }
        svg#colonyMap { width: 100%; height: 100%; }

        .log-console { font-family: monospace; font-size: 0.72rem; flex: 1; overflow-y: auto; background: rgba(0,0,0,0.6); padding: 8px; border-radius: 4px; display: flex; flex-direction: column; gap: 4px; }
        .log-entry { padding: 2px 4px; border-left: 2px solid var(--accent-cyan); }
        .log-entry.INFO { border-color: var(--accent-cyan); color: #94a3b8; }
        .log-entry.JOURNAL { border-color: var(--accent-green); color: #a7f3d0; }
        .log-entry.EVENT { border-color: var(--accent-purple); color: #e9d5ff; }
        .log-entry.ERROR { border-color: var(--accent-red); color: #fecdd3; }

        .pipeline-bar { display: flex; justify-content: space-between; gap: 4px; background: rgba(0,0,0,0.3); padding: 8px; border-radius: 6px; }
        .pipe-step { flex: 1; text-align: center; font-size: 0.65rem; padding: 4px; background: rgba(255,255,255,0.03); border-radius: 4px; color: var(--text-muted); border: 1px solid transparent; }
        .pipe-step.active { background: rgba(0,240,255,0.15); border-color: var(--accent-cyan); color: var(--accent-cyan); box-shadow: 0 0 8px rgba(0,240,255,0.3); }

        .modal-overlay { position: fixed; inset: 0; background: rgba(0,0,0,0.8); backdrop-filter: blur(8px); display: none; justify-content: center; align-items: center; z-index: 100; }
        .modal-card { width: 500px; background: #0b101d; border: 1px solid var(--accent-cyan); border-radius: 8px; padding: 20px; display: flex; flex-direction: column; gap: 14px; }
        input, select { background: rgba(0,0,0,0.5); border: 1px solid var(--border-card); color: #fff; padding: 8px; border-radius: 4px; width: 100%; }

        .badge { font-size: 0.65rem; padding: 2px 6px; border-radius: 3px; font-weight: bold; }
        .badge-idle { background: rgba(100,116,139,0.2); color: #94a3b8; }
        .badge-active { background: rgba(0,255,136,0.2); color: var(--accent-green); border: 1px solid var(--accent-green); }
    </style>
</head>
<body>

    <header>
        <div class="brand">
            <span>SPARK COLONY OS</span>
            <span class="brand-badge">MISSION CONTROL v5.0</span>
        </div>
        <div class="telemetry-bar">
            <div class="tele-item"><span class="tele-label">SYSTEM STATE</span><span class="tele-val" id="teleState">READY</span></div>
            <div class="tele-item"><span class="tele-label">UPTIME</span><span class="tele-val" id="teleUptime">0s</span></div>
            <div class="tele-item"><span class="tele-label">CPU / RAM</span><span class="tele-val" id="teleHardware">0% / 0%</span></div>
            <div class="tele-item"><span class="tele-label">ACTIVE MISSIONS</span><span class="tele-val" id="teleMissions">0</span></div>
            <div class="tele-item"><span class="tele-label">TOTAL COST</span><span class="tele-val" id="teleCost">$0.00 USD</span></div>
            <div class="tele-item"><span class="tele-label">WS STREAM</span><span class="tele-val" style="color:var(--accent-green);" id="teleWS">CONNECTED</span></div>
        </div>
    </header>

    <div class="dashboard-grid">
        <div class="glass-panel">
            <div class="panel-header">COMMAND & AGENT REGISTRY</div>
            <button class="command-btn" onclick="openDirectiveModal()">+ DISPATCH RESEARCH DIRECTIVE (Ctrl+K)</button>
            
            <div class="panel-header" style="margin-top:10px;">COLONY AGENT STATUS</div>
            <div id="agentList" style="display:flex; flex-direction:column; gap:8px; overflow-y:auto; flex:1;">
            </div>
        </div>

        <div class="glass-panel" style="grid-column: span 1;">
            <div class="panel-header">LIVE COLONY TOPOLOGY MAP</div>
            <div class="colony-map-container">
                <svg id="colonyMap" viewBox="0 0 600 320">
                    <line x1="300" y1="50" x2="150" y2="130" stroke="#00f0ff" stroke-width="1.5" stroke-dasharray="4" />
                    <line x1="300" y1="50" x2="300" y2="130" stroke="#00f0ff" stroke-width="1.5" stroke-dasharray="4" />
                    <line x1="300" y1="50" x2="450" y2="130" stroke="#00f0ff" stroke-width="1.5" stroke-dasharray="4" />
                    <line x1="150" y1="130" x2="220" y2="230" stroke="#00ff88" stroke-width="1.5" />
                    <line x1="300" y1="130" x2="220" y2="230" stroke="#00ff88" stroke-width="1.5" />
                    <line x1="450" y1="130" x2="380" y2="230" stroke="#9d00ff" stroke-width="1.5" />
                    
                    <g transform="translate(300,50)"><circle r="22" fill="#0d121f" stroke="#00f0ff" stroke-width="2"/><text y="4" text-anchor="middle" fill="#00f0ff" font-size="10" font-weight="bold">COMMANDER</text></g>
                    <g transform="translate(150,130)"><circle r="18" fill="#0d121f" stroke="#00ff88" stroke-width="2"/><text y="4" text-anchor="middle" fill="#e2e8f0" font-size="9">RESEARCH</text></g>
                    <g transform="translate(300,130)"><circle r="18" fill="#0d121f" stroke="#00ff88" stroke-width="2"/><text y="4" text-anchor="middle" fill="#e2e8f0" font-size="9">BROWSER</text></g>
                    <g transform="translate(450,130)"><circle r="18" fill="#0d121f" stroke="#9d00ff" stroke-width="2"/><text y="4" text-anchor="middle" fill="#e2e8f0" font-size="9">VISION</text></g>
                    <g transform="translate(140,230)"><circle r="18" fill="#0d121f" stroke="#ffb700" stroke-width="2"/><text y="4" text-anchor="middle" fill="#e2e8f0" font-size="9">FACT CHECK</text></g>
                    <g transform="translate(260,230)"><circle r="18" fill="#0d121f" stroke="#ffb700" stroke-width="2"/><text y="4" text-anchor="middle" fill="#e2e8f0" font-size="9">JUDGE</text></g>
                    <g transform="translate(380,230)"><circle r="18" fill="#0d121f" stroke="#00f0ff" stroke-width="2"/><text y="4" text-anchor="middle" fill="#e2e8f0" font-size="9">WRITER</text></g>
                    <g transform="translate(490,230)"><circle r="18" fill="#0d121f" stroke="#64748b" stroke-width="2"/><text y="4" text-anchor="middle" fill="#e2e8f0" font-size="9">MEMORY</text></g>
                </svg>
            </div>

            <div class="panel-header">REASONING & RESEARCH PIPELINE</div>
            <div class="pipeline-bar">
                <div class="pipe-step active" id="p1">1. PLAN</div>
                <div class="pipe-step" id="p2">2. SEARCH</div>
                <div class="pipe-step" id="p3">3. BROWSE</div>
                <div class="pipe-step" id="p4">4. VERIFY</div>
                <div class="pipe-step" id="p5">5. REASON</div>
                <div class="pipe-step" id="p6">6. SYNTHESIZE</div>
                <div class="pipe-step" id="p7">7. ARCHIVE</div>
            </div>
        </div>

        <div class="glass-panel">
            <div class="panel-header">BROWSER & VISION TELECAST</div>
            <div style="height:140px; background:#000; border:1px solid rgba(255,255,255,0.1); border-radius:4px; padding:10px; font-size:0.75rem;">
                <div style="color:var(--accent-green); margin-bottom:4px;">VIEWPORT: Active Headless Stream</div>
                <div id="browserStreamText" style="color:#94a3b8;">No active browser session observed. Standby...</div>
            </div>

            <div class="panel-header" style="margin-top:10px;">LIVE SYSTEM LOG CONSOLE</div>
            <div class="log-console" id="logConsole">
                <div class="log-entry INFO">[00:00:00] Spark Colony OS v5.0 Kernel Online.</div>
            </div>
        </div>
    </div>

    <div class="modal-overlay" id="directiveModal">
        <div class="modal-card">
            <div class="panel-header">DISPATCH RESEARCH DIRECTIVE</div>
            <div>
                <label style="font-size:0.75rem; color:var(--text-muted);">Research Directive / Topic</label>
                <input type="text" id="topicInput" placeholder="e.g. Decentralized Multi-Agent AI Architectures">
            </div>
            <div style="display:flex; gap:10px;">
                <div style="flex:1;">
                    <label style="font-size:0.75rem; color:var(--text-muted);">Thinking Mode</label>
                    <select id="modeSelect">
                        <option value="Research">Research (Deep Search)</option>
                        <option value="Analytical">Analytical (Source Compare)</option>
                        <option value="Fast">Fast (Low Cost)</option>
                        <option value="Critical">Critical (Disprove)</option>
                        <option value="Scientific">Scientific (Pyramid Authority)</option>
                    </select>
                </div>
                <div style="flex:1;">
                    <label style="font-size:0.75rem; color:var(--text-muted);">Max Depth</label>
                    <input type="number" id="depthInput" value="3" min="1" max="10">
                </div>
            </div>
            <div style="display:flex; justify-content:flex-end; gap:10px; margin-top:10px;">
                <button style="padding:8px 16px; background:transparent; border:1px solid var(--text-muted); color:#fff; border-radius:4px; cursor:pointer;" onclick="closeDirectiveModal()">Cancel</button>
                <button style="padding:8px 16px; background:var(--accent-cyan); border:none; color:#000; font-weight:bold; border-radius:4px; cursor:pointer;" onclick="submitMission()">Dispatch Mission (Ctrl+Enter)</button>
            </div>
        </div>
    </div>

    <script>
        const ws = new WebSocket(`ws://${location.host}/api/v1/ws`);
        
        ws.onmessage = (event) => {
            const msg = JSON.parse(event.data);
            appendLog(msg.event_type, JSON.stringify(msg.data));
            if (msg.event_type === "AgentStateChanged") fetchStatus();
            if (msg.event_type === "MissionCreated" || msg.event_type === "MissionFinished") fetchStatus();
        };

        async function fetchStatus() {
            try {
                const res = await fetch('/api/v1/system/status');
                const data = await res.json();
                
                document.getElementById('teleState').innerText = data.app_state;
                document.getElementById('teleUptime').innerText = `${Math.round(data.uptime_seconds)}s`;
                document.getElementById('teleHardware').innerText = `${data.cpu_usage_percent}% / ${data.memory_usage_percent}%`;
                document.getElementById('teleMissions').innerText = data.total_missions;
                document.getElementById('teleCost').innerText = `$${data.total_cost_usd.toFixed(4)} USD`;

                renderAgents(data.agents);
            } catch (e) {
                console.error("Failed fetching status", e);
            }
        }

        function renderAgents(agents) {
            const list = document.getElementById('agentList');
            list.innerHTML = '';
            agents.forEach(a => {
                const div = document.createElement('div');
                div.style.padding = '8px';
                div.style.background = 'rgba(0,0,0,0.3)';
                div.style.borderRadius = '4px';
                div.style.border = '1px solid rgba(255,255,255,0.05)';
                div.innerHTML = `
                    <div style="display:flex; justify-content:space-between; align-items:center;">
                        <span style="font-size:0.8rem; font-weight:bold; color:#fff;">${a.name}</span>
                        <span class="badge ${a.state === 'Idle' ? 'badge-idle' : 'badge-active'}">${a.state}</span>
                    </div>
                    <div style="font-size:0.68rem; color:var(--text-muted); margin-top:2px;">${a.current_task || 'Awaiting directive'}</div>
                `;
                list.appendChild(div);
            });
        }

        function appendLog(type, text) {
            const consoleBox = document.getElementById('logConsole');
            const entry = document.createElement('div');
            const timeStr = new Date().toLocaleTimeString();
            entry.className = `log-entry ${type.includes('Error') ? 'ERROR' : type.includes('Journal') ? 'JOURNAL' : 'INFO'}`;
            entry.innerText = `[${timeStr}] [${type}] ${text}`;
            consoleBox.appendChild(entry);
            consoleBox.scrollTop = consoleBox.scrollHeight;
        }

        function openDirectiveModal() { document.getElementById('directiveModal').style.display = 'flex'; document.getElementById('topicInput').focus(); }
        function closeDirectiveModal() { document.getElementById('directiveModal').style.display = 'none'; }

        async function submitMission() {
            const topic = document.getElementById('topicInput').value;
            const mode = document.getElementById('modeSelect').value;
            const depth = parseInt(document.getElementById('depthInput').value);
            if (!topic) return;

            closeDirectiveModal();
            await fetch('/api/v1/missions', {
                method: 'POST',
                headers: { 'Content-Type': 'application/json' },
                body: JSON.stringify({ topic: topic, thinking_mode: mode, max_depth: depth })
            });
            document.getElementById('topicInput').value = '';
            fetchStatus();
        }

        document.addEventListener('keydown', (e) => {
            if (e.ctrlKey && e.key === 'k') { e.preventDefault(); openDirectiveModal(); }
            if (e.key === 'Escape') closeDirectiveModal();
            if (e.ctrlKey && e.key === 'Enter') submitMission();
        });

        setInterval(fetchStatus, 3000);
        fetchStatus();
    </script>
</body>
</html>"""


# --- HTML DASHBOARD ENDPOINT ---
@app.get("/", response_class=HTMLResponse)
async def get_mission_control_ui():
    """Serve NASA Mission Control Operator Dashboard Template."""
    if os.path.exists("templates/index.html"):
        with open("templates/index.html", "r", encoding="utf-8") as f:
            return HTMLResponse(content=f.read())
    return HTMLResponse(content=MISSION_CONTROL_HTML)


# --- SYSTEM APIs ---
@app.get("/api/v1/system/status", response_model=SystemStatusResponse)
async def system_status():
    """Diagnostic readout showing active agents, OS state, hardware metrics, and total cost."""
    metrics = colony_os.monitor.get_metrics()
    agents_raw = await colony_os.db.get_all_agents()
    agents = [
        AgentStatusModel(
            agent_id=a["id"],
            name=a["name"],
            role=a["role"],
            state=AgentState(a["state"]),
            current_task=a["current_task"],
            confidence=a["confidence"],
            last_active=a["last_active"],
            enabled=bool(a["enabled"]),
            capabilities=colony_os.agent_registry[a["id"]].capability_profile.capabilities if a["id"] in colony_os.agent_registry else [],
            tools=colony_os.agent_registry[a["id"]].capability_profile.tools if a["id"] in colony_os.agent_registry else [],
            current_load=colony_os.agent_registry[a["id"]].capability_profile.current_load if a["id"] in colony_os.agent_registry else 0.0,
            is_dynamic=colony_os.agent_registry[a["id"]].is_dynamic if a["id"] in colony_os.agent_registry else False,
        )
        for a in agents_raw
    ]
    total_missions = await colony_os.db.count_missions()
    total_memories = await colony_os.db.count_memories()
    total_messages = await colony_os.db.count_messages()
    total_cost = await colony_os.db.get_total_system_cost()

    return SystemStatusResponse(
        app_state=colony_os.app_state,
        uptime_seconds=metrics["uptime_seconds"],
        active_agents=len([a for a in agents if a.state != AgentState.SLEEPING and a.state != AgentState.IDLE]),
        total_missions=total_missions,
        total_memories=total_memories,
        total_messages=total_messages,
        total_cost_usd=round(total_cost, 6),
        cpu_usage_percent=metrics["cpu_usage_percent"],
        memory_usage_percent=metrics["memory_usage_percent"],
        agents=agents,
    )


@app.get("/api/v1/system/metrics")
async def system_metrics():
    """Live hardware performance readout."""
    return colony_os.monitor.get_metrics()


@app.get("/api/v1/system/version")
async def system_version():
    """Return OS subsystem version matrix."""
    return VersionEngine.get_version_info()


@app.post("/api/v1/system/backup")
async def trigger_backup():
    """Trigger automated timestamped database backup."""
    backup_path = BackupEngine.perform_backup()
    return {"status": "SUCCESS", "backup_file": backup_path}


# --- MISSION APIs ---
@app.post("/api/v1/missions", response_model=MissionResponse, status_code=status.HTTP_202_ACCEPTED)
async def create_mission(request: CreateMissionRequest, background_tasks: BackgroundTasks):
    """Receive research directive and dispatch autonomous execution."""
    mission_id = f"mission-{uuid.uuid4().hex[:12]}"
    await colony_os.db.save_mission(mission_id, request.topic, MissionStatus.INITIALIZING, DecisionStage.RECEIVE_MISSION)
    await colony_os.event_bus.emit("MissionCreated", mission_id, "API", {"topic": request.topic, "mode": request.thinking_mode.value})

    background_tasks.add_task(
        colony_os.commander.execute_mission_pipeline,
        mission_id=mission_id,
        topic=request.topic,
        max_depth=request.max_depth,
        research_agent=colony_os.researcher,
        fact_checker=colony_os.fact_checker,
        writer=colony_os.writer,
        evidence_judge=colony_os.evidence_judge,
        thinking_mode=request.thinking_mode,
    )

    return MissionResponse(
        mission_id=mission_id,
        topic=request.topic,
        status=MissionStatus.INITIALIZING,
        current_stage=DecisionStage.RECEIVE_MISSION,
        created_at=datetime.now(timezone.utc).isoformat(),
        message=f"Mission registered [{request.thinking_mode.value} Mode] and submitted to Commander Prime.",
    )


@app.get("/api/v1/missions", response_model=List[Dict[str, Any]])
async def list_missions():
    """Retrieve history of all colony missions."""
    return await colony_os.db.get_all_missions()


@app.get("/api/v1/missions/{mission_id}", response_model=MissionDetailResponse)
async def get_mission_detail(mission_id: str):
    """Fetch complete mission details including messages, journals, memories, and evidence."""
    mission = await colony_os.db.get_mission(mission_id)
    if not mission:
        raise HTTPException(status_code=404, detail=f"Mission '{mission_id}' not found.")

    journals = await colony_os.db.get_journals_for_mission(mission_id)
    memories = await colony_os.db.get_memories_for_mission(mission_id)
    messages = await colony_os.db.get_messages_for_mission(mission_id)
    evidence = await colony_os.db.get_evidence_for_mission(mission_id)

    return MissionDetailResponse(
        mission_id=mission["id"],
        topic=mission["topic"],
        status=MissionStatus(mission["status"]),
        stage=DecisionStage(mission["stage"]),
        created_at=mission["created_at"],
        updated_at=mission["updated_at"],
        summary=mission["summary"],
        total_cost_usd=round(mission["total_cost"], 6),
        messages=messages,
        journals=journals,
        memories=memories,
        evidence=evidence,
    )


@app.get("/api/v1/missions/{mission_id}/checkpoints")
async def list_checkpoints(mission_id: str):
    """Retrieve saved state checkpoints for a mission."""
    return await colony_os.db.get_checkpoints_for_mission(mission_id)


@app.post("/api/v1/missions/{mission_id}/recover")
async def recover_mission(mission_id: str, background_tasks: BackgroundTasks):
    """Resume execution of a failed/paused mission from its last valid checkpoint."""
    checkpoint = await RecoveryEngine.recover_mission(colony_os.db, mission_id)
    if not checkpoint:
        raise HTTPException(status_code=404, detail="No checkpoint found for recovery.")

    mission = await colony_os.db.get_mission(mission_id)
    background_tasks.add_task(
        colony_os.commander.execute_mission_pipeline,
        mission_id=mission_id,
        topic=mission["topic"],
        max_depth=3,
        research_agent=colony_os.researcher,
        fact_checker=colony_os.fact_checker,
        writer=colony_os.writer,
        evidence_judge=colony_os.evidence_judge,
    )
    return {"message": f"Mission {mission_id} resumed from stage {checkpoint['stage']}"}


@app.get("/api/v1/missions/{mission_id}/replay")
async def mission_replay(mission_id: str):
    """Retrieve complete chronological time-series trace for replaying a mission."""
    mission = await colony_os.db.get_mission(mission_id)
    if not mission:
        raise HTTPException(status_code=404, detail="Mission not found.")

    journals = await colony_os.db.get_journals_for_mission(mission_id)
    messages = await colony_os.db.get_messages_for_mission(mission_id)
    checkpoints = await colony_os.db.get_checkpoints_for_mission(mission_id)

    return {"mission": mission, "timeline": {"journals": journals, "messages": messages, "checkpoints": checkpoints}}


@app.post("/api/v1/missions/{mission_id}/pause")
async def pause_mission(mission_id: str):
    """Pause active mission execution."""
    mission = await colony_os.db.get_mission(mission_id)
    if not mission:
        raise HTTPException(status_code=404, detail="Mission not found.")
    await colony_os.db.update_mission_status(mission_id, MissionStatus.PAUSED)
    await colony_os.event_bus.emit("MissionUpdated", mission_id, "System", {"status": "PAUSED"})
    return {"message": f"Mission {mission_id} paused."}


@app.post("/api/v1/missions/{mission_id}/resume")
async def resume_mission(mission_id: str):
    """Resume paused mission."""
    mission = await colony_os.db.get_mission(mission_id)
    if not mission:
        raise HTTPException(status_code=404, detail="Mission not found.")
    await colony_os.db.update_mission_status(mission_id, MissionStatus.IN_PROGRESS)
    await colony_os.event_bus.emit("MissionUpdated", mission_id, "System", {"status": "IN_PROGRESS"})
    return {"message": f"Mission {mission_id} resumed."}


@app.post("/api/v1/missions/{mission_id}/cancel")
async def cancel_mission(mission_id: str):
    """Cancel mission."""
    mission = await colony_os.db.get_mission(mission_id)
    if not mission:
        raise HTTPException(status_code=404, detail="Mission not found.")
    await colony_os.db.update_mission_status(mission_id, MissionStatus.CANCELLED)
    await colony_os.event_bus.emit("MissionUpdated", mission_id, "System", {"status": "CANCELLED"})
    return {"message": f"Mission {mission_id} cancelled."}


@app.delete("/api/v1/missions/{mission_id}")
async def delete_mission(mission_id: str):
    """Delete mission and associated records."""
    mission = await colony_os.db.get_mission(mission_id)
    if not mission:
        raise HTTPException(status_code=404, detail="Mission not found.")
    await colony_os.db.delete_mission(mission_id)
    return {"message": f"Mission {mission_id} deleted."}


@app.post("/api/v1/missions/{mission_id}/clone", response_model=MissionResponse, status_code=status.HTTP_202_ACCEPTED)
async def clone_mission(mission_id: str, background_tasks: BackgroundTasks):
    """Clone an existing mission and re-trigger execution."""
    mission = await colony_os.db.get_mission(mission_id)
    if not mission:
        raise HTTPException(status_code=404, detail="Mission not found.")
    new_request = CreateMissionRequest(topic=mission["topic"])
    return await create_mission(new_request, background_tasks)


@app.post("/api/v1/missions/{mission_id}/restart", response_model=MissionResponse, status_code=status.HTTP_202_ACCEPTED)
async def restart_mission(mission_id: str, background_tasks: BackgroundTasks):
    """Restart a failed or completed mission."""
    mission = await colony_os.db.get_mission(mission_id)
    if not mission:
        raise HTTPException(status_code=404, detail="Mission not found.")
    await colony_os.db.update_mission_status(mission_id, MissionStatus.INITIALIZING, DecisionStage.RECEIVE_MISSION)
    background_tasks.add_task(
        colony_os.commander.execute_mission_pipeline,
        mission_id=mission_id,
        topic=mission["topic"],
        max_depth=3,
        research_agent=colony_os.researcher,
        fact_checker=colony_os.fact_checker,
        writer=colony_os.writer,
        evidence_judge=colony_os.evidence_judge,
    )
    return MissionResponse(
        mission_id=mission_id,
        topic=mission["topic"],
        status=MissionStatus.INITIALIZING,
        current_stage=DecisionStage.RECEIVE_MISSION,
        created_at=datetime.now(timezone.utc).isoformat(),
        message="Mission restarted.",
    )


# --- AGENT APIs ---
@app.get("/api/v1/agents", response_model=List[AgentStatusModel])
async def list_agents():
    """List status and confidence metrics for all registered colony agents."""
    agents = await colony_os.db.get_all_agents()
    return [
        AgentStatusModel(
            agent_id=a["id"],
            name=a["name"],
            role=a["role"],
            state=AgentState(a["state"]),
            current_task=a["current_task"],
            confidence=a["confidence"],
            last_active=a["last_active"],
            enabled=bool(a["enabled"]),
            capabilities=colony_os.agent_registry[a["id"]].capability_profile.capabilities if a["id"] in colony_os.agent_registry else [],
            tools=colony_os.agent_registry[a["id"]].capability_profile.tools if a["id"] in colony_os.agent_registry else [],
            current_load=colony_os.agent_registry[a["id"]].capability_profile.current_load if a["id"] in colony_os.agent_registry else 0.0,
            is_dynamic=colony_os.agent_registry[a["id"]].is_dynamic if a["id"] in colony_os.agent_registry else False,
        )
        for a in agents
    ]


@app.get("/api/v1/agents/{agent_id}")
async def get_agent_details(agent_id: str):
    """Fetch details for a specific agent."""
    agent = await colony_os.db.get_agent(agent_id)
    if not agent:
        raise HTTPException(status_code=404, detail="Agent not found.")
    return agent


@app.post("/api/v1/agents/{agent_id}/restart")
async def restart_agent(agent_id: str):
    """Reset agent state to IDLE."""
    if agent_id in colony_os.agent_registry:
        agent = colony_os.agent_registry[agent_id]
        await agent.set_state(AgentState.IDLE, current_task=None, confidence=100.0)
        return {"message": f"Agent '{agent.name}' state reset to IDLE."}
    raise HTTPException(status_code=404, detail="Agent not found.")


@app.post("/api/v1/agents/{agent_id}/toggle")
async def toggle_agent(agent_id: str, enable: bool = True):
    """Enable or disable a colony agent."""
    if agent_id in colony_os.agent_registry:
        agent = colony_os.agent_registry[agent_id]
        agent.enabled = enable
        await agent.set_state(AgentState.IDLE if enable else AgentState.SLEEPING, current_task=None if enable else "Disabled")
        return {"message": f"Agent '{agent.name}' set to enabled={enable}."}
    raise HTTPException(status_code=404, detail="Agent not found.")


# --- MEMORY APIs ---
@app.get("/api/v1/memory/search")
async def search_memory(q: str = Query(..., min_length=1), tag: Optional[str] = None):
    """Search vector/SQLite memory store by query string or tag."""
    return await colony_os.db.search_memories(q, tag)


@app.post("/api/v1/memory")
async def add_memory(req: AddMemoryRequest):
    """Manually add a memory record into the system store."""
    mem_id = await colony_os.db.save_memory(req.agent_id, req.mission_id, req.content, req.tags)
    return {"memory_id": mem_id, "status": "SUCCESS"}


@app.delete("/api/v1/memory/{memory_id}")
async def delete_memory(memory_id: str):
    """Delete a memory entry by ID."""
    await colony_os.db.delete_memory(memory_id)
    return {"message": f"Memory {memory_id} deleted."}


@app.post("/api/v1/memory/cache-page")
async def cache_browser_page(req: CachePageRequest):
    """Cache raw HTML content from browser visits."""
    cache_id = await colony_os.db.save_browser_cache(req.url, req.html, req.title)
    return {"cache_id": cache_id, "url": req.url, "status": "CACHED"}


@app.delete("/api/v1/memory/cache")
async def clear_browser_cache():
    """Clear all cached browser page entries."""
    await colony_os.db.clear_browser_cache()
    return {"message": "Browser cache cleared successfully."}


@app.post("/api/v1/memory/consolidate")
async def consolidate_memory():
    """Trigger memory deduplication and cache pruning."""
    return await MemoryConsolidationEngine.consolidate_memories(colony_os.db)


# --- RESEARCH & KNOWLEDGE GRAPH APIs ---
@app.get("/api/v1/research/evidence/{mission_id}")
async def get_mission_evidence(mission_id: str):
    """Retrieve scored evidence claims for a specific research mission."""
    return await colony_os.db.get_evidence_for_mission(mission_id)


@app.get("/api/v1/research/confidence/{mission_id}")
async def get_mission_confidence_report(mission_id: str):
    """Generate composite confidence summary report for mission claims."""
    evidence = await colony_os.db.get_evidence_for_mission(mission_id)
    if not evidence:
        return {"mission_id": mission_id, "average_confidence": 0.0, "total_claims": 0}
    avg = sum(e["confidence"] for e in evidence) / len(evidence)
    return {"mission_id": mission_id, "average_confidence": round(avg, 2), "total_claims": len(evidence), "claims": evidence}


@app.get("/api/v1/research/reflections/{mission_id}")
async def get_mission_reflections(mission_id: str):
    """Retrieve post-mission reflections and lessons learned for a specific mission."""
    return await colony_os.db.get_reflections_for_mission(mission_id)


@app.get("/api/v1/research/reflections")
async def list_all_reflections():
    """List system-wide colony reflections and learned experience records."""
    return await colony_os.db.get_all_reflections()


@app.get("/api/v1/research/knowledge-graph/{mission_id}")
async def get_knowledge_graph(mission_id: str):
    """Retrieve Knowledge Graph entities and relational connections for a mission."""
    return await colony_os.db.get_knowledge_graph(mission_id)


@app.post("/api/v1/research/decision-score", response_model=DecisionScore)
async def compute_decision_score(benefit: float, cost: float, risk: float, confidence: float, resource_usage: float):
    """Calculate expected value decision score for potential agent actions."""
    return DecisionScoreEngine.calculate_action_score(benefit, cost, risk, confidence, resource_usage)


# --- BROWSER INTELLIGENCE APIs ---
@app.get("/api/v1/browser/pool")
async def get_browser_pool_status():
    """Retrieve status and health metrics for the browser pool."""
    return colony_os.browser_pool.get_status()


@app.post("/api/v1/browser/observe", response_model=PageObservation)
async def observe_webpage(req: ObservePageRequest):
    """Observe target URL, extract main content, and detect popups/CAPTCHAs."""
    return await colony_os.browser.observe_page(req.mission_id, req.url)


@app.post("/api/v1/browser/navigate")
async def navigate_webpage(req: NavigatePageRequest):
    """Execute self-healing web navigation and action."""
    return await colony_os.browser.navigate_with_healing(req.mission_id, req.url, req.target_action)


@app.get("/api/v1/browser/profiles")
async def list_website_profiles():
    """Retrieve domain trust and interaction profiles learned by the colony."""
    return await colony_os.db.get_website_profiles()


@app.get("/api/v1/browser/downloads")
async def list_downloaded_files():
    """List files downloaded and processed by browser sessions."""
    return await colony_os.db.get_downloaded_files()


# --- VISION APIs ---
@app.post("/api/v1/vision/analyze-layout", response_model=VisionOutput)
async def vision_analyze_layout(image_ref: str = "screen_sample.png", mission_id: str = "manual"):
    """Generate comprehensive structured visual output for a target screenshot."""
    return await colony_os.vision.analyze_screen(mission_id, image_ref)


@app.post("/api/v1/vision/diff", response_model=PageDiffResult)
async def vision_compute_diff(prev_url: str, curr_url: str):
    """Calculate structural and visual diff between two page states."""
    return PageDiffEngine.compute_diff(prev_url, curr_url)


# --- CONFIGURATION APIs ---
@app.get("/api/v1/config")
async def get_config():
    """Retrieve runtime colony configuration parameters."""
    return colony_os.config.to_dict()


@app.post("/api/v1/config")
async def update_config(req: UpdateConfigRequest):
    """Update runtime colony configuration dynamically."""
    return colony_os.config.update(req)


# --- PLUGIN APIs ---
@app.get("/api/v1/plugins")
async def list_plugins():
    """List registered dynamic plugins."""
    return await colony_os.plugin_manager.list_plugins()


@app.post("/api/v1/plugins/register")
async def register_plugin(req: RegisterPluginRequest):
    """Register a new plugin with the Colony OS Plugin Manager."""
    plugin_id = await colony_os.plugin_manager.register_plugin(req)
    return {"plugin_id": plugin_id, "status": "REGISTERED"}


# --- HEALTH API ---
@app.get("/api/v1/health")
async def health_check():
    """Perform full diagnostic health check across sub-systems."""
    db_ok = True
    try:
        await colony_os.db.count_missions()
    except Exception:
        db_ok = False

    return {
        "status": "HEALTHY" if db_ok and colony_os.app_state == AppState.READY else "DEGRADED",
        "app_state": colony_os.app_state,
        "database": "OK" if db_ok else "ERROR",
        "browser_pool": colony_os.browser_pool.get_status()["total_browsers"],
        "websocket_clients": len(colony_os.ws_manager.active_connections),
        "scheduler_queue": colony_os.scheduler.queue_size(),
        "registered_agents": len(colony_os.agent_registry),
        "version": VersionEngine.get_version_info()["kernel_version"],
        "timestamp": datetime.now(timezone.utc).isoformat(),
    }


# --- V2 COLONY RUNTIME APIs ---
@app.get("/api/v1/runtime/status")
async def runtime_status():
    """Retrieve full V2 Colony Runtime status including dynamic agents, active contexts, and blackboard count."""
    active_workers = [
        DynamicAgentInfo(
            agent_id=a.agent_id,
            name=a.name,
            role=a.role,
            mission_id=a.assigned_mission_id,
            is_dynamic=True,
            created_at=datetime.now(timezone.utc).isoformat(),
        )
        for a in colony_os.runtime.dynamic_agents.values()
    ]
    contexts = await colony_os.db.get_all_mission_contexts()

    return {
        "status": "ONLINE",
        "active_dynamic_workers_count": len(active_workers),
        "dynamic_workers": active_workers,
        "active_mission_contexts_count": len(contexts),
        "active_mission_contexts": contexts,
    }


@app.get("/api/v1/runtime/resources", response_model=ResourceMetricsModel)
async def runtime_resources():
    """Retrieve deep telemetry on system hardware, LLM usage, browser pool, and task queue."""
    return colony_os.runtime.resource_manager.get_resource_metrics(colony_os.browser_pool, colony_os.scheduler)


@app.post("/api/v1/runtime/agents/spawn", response_model=DynamicAgentInfo)
async def spawn_dynamic_agent(req: SpawnAgentRequest):
    """Dynamically spawn a temporary worker agent for a specific mission."""
    worker = await colony_os.runtime.spawn_worker(req.role, req.mission_id, req.capabilities, req.tools)
    return DynamicAgentInfo(
        agent_id=worker.agent_id,
        name=worker.name,
        role=worker.role,
        mission_id=worker.assigned_mission_id,
        is_dynamic=True,
        created_at=datetime.now(timezone.utc).isoformat(),
    )


@app.delete("/api/v1/runtime/agents/{agent_id}")
async def despawn_dynamic_agent(agent_id: str):
    """Manually despawn a dynamic worker agent."""
    if agent_id in colony_os.runtime.dynamic_agents:
        worker = colony_os.runtime.dynamic_agents.pop(agent_id)
        await worker.set_state(AgentState.SLEEPING, current_task="Despawned")
        return {"message": f"Dynamic agent {worker.name} despawned."}
    raise HTTPException(status_code=404, detail="Dynamic agent not found.")


@app.get("/api/v1/runtime/agents/capabilities")
async def get_agent_capability_matrix():
    """Expose agent capability profiles, current load, tools, and experience across static and dynamic agents."""
    matrix = []
    all_agents = {**colony_os.agent_registry, **colony_os.runtime.dynamic_agents}
    for agent in all_agents.values():
        matrix.append(
            {
                "agent_id": agent.agent_id,
                "name": agent.name,
                "role": agent.role,
                "is_dynamic": agent.is_dynamic,
                "capabilities": agent.capability_profile.capabilities,
                "tools": agent.capability_profile.tools,
                "current_model": agent.capability_profile.current_model,
                "experience": agent.capability_profile.experience,
                "current_load": agent.capability_profile.current_load,
                "confidence": agent.confidence,
            }
        )
    return matrix


@app.get("/api/v1/runtime/blackboard/{mission_id}")
async def get_blackboard_observations(mission_id: str, topic: Optional[str] = None):
    """Retrieve shared blackboard observations published during mission execution."""
    return await colony_os.runtime.blackboard.query(mission_id, topic)


@app.get("/api/v1/runtime/missions/contexts")
async def get_parallel_mission_contexts():
    """Retrieve parallel mission context tracking objects across active executions."""
    return await colony_os.db.get_all_mission_contexts()


# --- V2 PHASE 2 UNIVERSAL MODEL & KEY ROTATION APIs ---
@app.get("/api/v1/models/telemetry", response_model=ModelTelemetryModel)
async def model_telemetry():
    """Retrieve Model Manager logical mappings and API Key rotation engine health metrics."""
    keys_tel = colony_os.runtime.key_rotator.get_telemetry()
    return ModelTelemetryModel(
        logical_models=colony_os.runtime.model_manager.logical_models,
        total_keys_managed=len(keys_tel),
        active_keys_count=len([k for k in keys_tel if not k.is_disabled and not k.is_busy]),
        disabled_keys_count=len([k for k in keys_tel if k.is_disabled]),
        total_model_calls=colony_os.runtime.model_manager.total_calls,
        key_telemetry=keys_tel,
    )


@app.post("/api/v1/models/generate")
async def generate_model_response(logical_model: LogicalModel, prompt: str, system_prompt: Optional[str] = None):
    """Execute LLM generation via Universal Model Manager with key rotation."""
    return await colony_os.runtime.model_manager.generate_response(logical_model, prompt, system_prompt)


# --- V2 PHASE 2 TOOL PERMISSION APIs ---
@app.get("/api/v1/tools/catalog", response_model=List[ToolDefinition])
async def list_tools_catalog():
    """Retrieve available colony tools and safety levels."""
    return list(colony_os.runtime.tool_engine.catalog.values())


@app.post("/api/v1/tools/request", response_model=ToolResponse)
async def request_tool_execution(req: ToolRequest):
    """Evaluate and request execution permission for a colony tool."""
    return colony_os.runtime.tool_engine.evaluate_request(req)


# --- V2 PHASE 2 LIVE BROWSER STREAMING APIs ---
@app.get("/api/v1/browser/stream/{mission_id}")
async def get_browser_mission_stream(mission_id: str):
    """Fetch latest base64 screenshot frame for live browser mission streaming."""
    frame = colony_os.browser_pool.get_mission_screenshot(mission_id)
    if not frame:
        return {"mission_id": mission_id, "has_stream": False, "frame_base64": None}
    return {"mission_id": mission_id, "has_stream": True, "frame_base64": frame}


@app.get("/api/v1/browser/thumbnails")
async def get_browser_thumbnails():
    """Fetch active browser viewport thumbnails across all sessions."""
    return colony_os.browser_pool.get_all_thumbnails()


# --- V2 PHASE 2 REPLAYABLE MISSION TIMELINE APIs ---
@app.get("/api/v1/missions/{mission_id}/timeline")
async def get_mission_timeline(mission_id: str):
    """Retrieve full replayable timeline of execution steps for a mission."""
    return await colony_os.db.get_mission_timeline(mission_id)


# --- V2 PHASE 3 COLLABORATION & DISCUSSION APIs ---
@app.post("/api/v1/runtime/discussions", response_model=DiscussionEntry)
async def post_discussion_entry(
    mission_id: str,
    agent_id: str,
    agent_name: str,
    discussion_type: DiscussionType,
    topic: str,
    content: str,
    evidence_ref: Optional[str] = None,
):
    """Publish a structured discussion entry (agree, disagree, criticize, suggest, etc.) to the colony."""
    return await colony_os.runtime.discussion_engine.post_discussion(
        mission_id=mission_id, agent_id=agent_id, agent_name=agent_name, discussion_type=discussion_type, topic=topic, content=content, evidence_ref=evidence_ref
    )


@app.get("/api/v1/runtime/discussions/{mission_id}")
async def get_mission_discussions(mission_id: str, tag: Optional[str] = None):
    """Retrieve all structured discussions published for a specific mission."""
    return await colony_os.runtime.discussion_engine.get_discussions(mission_id, tag)


# --- V2 PHASE 3 DEBATE & CONSENSUS APIs ---
@app.post("/api/v1/runtime/debates/initiate", response_model=DebateSession)
async def initiate_debate_session(mission_id: str, topic: str, participant_ids: List[str]):
    """Initiate a structured multi-agent debate session on a topic."""
    participants = []
    for pid in participant_ids:
        if pid in colony_os.agent_registry:
            a = colony_os.agent_registry[pid]
            participants.append({"id": a.agent_id, "name": a.name})
        elif pid in colony_os.runtime.dynamic_agents:
            a = colony_os.runtime.dynamic_agents[pid]
            participants.append({"id": a.agent_id, "name": a.name})

    if not participants:
        participants = [
            {"id": colony_os.researcher.agent_id, "name": colony_os.researcher.name},
            {"id": colony_os.fact_checker.agent_id, "name": colony_os.fact_checker.name},
        ]

    return await colony_os.runtime.debate_engine.initiate_debate(mission_id, topic, participants)


@app.get("/api/v1/runtime/debates/{mission_id}")
async def get_mission_debates(mission_id: str):
    """Fetch all debate sessions associated with a mission."""
    return await colony_os.db.get_debates_for_mission(mission_id)


@app.post("/api/v1/runtime/consensus/calculate", response_model=ConsensusResult)
async def calculate_consensus_score(req: ConsensusRequest):
    """Calculate composite consensus score across 7 intelligence vectors."""
    return colony_os.runtime.consensus_engine.calculate_consensus(req)


# --- V2 PHASE 3 TASK NEGOTIATION APIs ---
@app.post("/api/v1/runtime/negotiate", response_model=TaskNegotiationResult)
async def negotiate_task_assignment(proposal: TaskNegotiationProposal):
    """Evaluate task negotiation proposal to eliminate duplicate work."""
    return await colony_os.runtime.negotiation_engine.evaluate_proposal(proposal)


# --- V2 PHASE 3 REPUTATION & REFLECTION APIs ---
@app.get("/api/v1/runtime/reputation")
async def get_all_agent_reputations():
    """Retrieve reputation, trust scores, experience, and accuracy rates across all colony agents."""
    return await colony_os.db.get_all_agent_reputations()


@app.get("/api/v1/runtime/reflections/summary/{mission_id}")
async def get_mission_reflections_v2(mission_id: str):
    """Retrieve detailed multi-agent post-mission reflections (what worked, failed, surprised, to improve)."""
    return await colony_os.db.get_agent_reflections_for_mission(mission_id)


@app.get("/api/v1/runtime/collective-memory/search")
async def search_collective_memory(query: str = Query(..., min_length=1), tag: Optional[str] = None):
    """Search unified collective memory containing discussions, debates, reflections, and knowledge updates."""
    return await colony_os.db.search_memories(query, tag)


# --- V2 PHASE 4 CONVERSATION & HUMAN-IN-THE-LOOP APIs ---
@app.post("/api/v1/conversation/chat")
async def chat_with_conversation_agent(req: ChatRequest):
    """Chat directly with the Colony Conversation Agent (Mnemosyne Chat)."""
    reply = await colony_os.runtime.conversation_agent.process_user_message(req.message, req.user_id or "human-operator")
    return {"response": reply, "agent": colony_os.runtime.conversation_agent.name}


@app.get("/api/v1/conversation/history")
async def get_conversation_chat_history(user_id: str = "human-operator", limit: int = 50):
    """Retrieve persistent conversation history for human operator."""
    return await colony_os.db.get_conversation_history(user_id, limit)


@app.get("/api/v1/approvals/pending")
async def get_pending_human_approvals():
    """List pending human-in-the-loop approval requests."""
    return await colony_os.db.get_pending_approvals()


@app.post("/api/v1/approvals/{approval_id}/resolve")
async def resolve_human_approval(approval_id: str, req: ResolveApprovalRequest):
    """Approve or reject a pending human approval request and resume mission execution."""
    approval = await colony_os.db.get_approval_request(approval_id)
    if not approval:
        raise HTTPException(status_code=404, detail="Approval request not found.")

    status_val = ApprovalStatus.APPROVED if req.approved else ApprovalStatus.REJECTED
    approval_obj = ApprovalRequestModel(
        id=approval["id"],
        mission_id=approval["mission_id"],
        agent_id=approval["agent_id"],
        action_type=ActionType(approval["action_type"]),
        prompt_message=approval["prompt_message"],
        status=status_val,
        input_data=req.input_data or {},
    )
    await colony_os.db.save_approval_request(approval_obj)

    if req.approved:
        await colony_os.db.update_mission_status(approval["mission_id"], MissionStatus.IN_PROGRESS)
        await colony_os.event_bus.emit("MissionResumed", approval["mission_id"], "HumanOperator", {"approval_id": approval_id})

    return {"status": status_val.value, "approval_id": approval_id, "mission_id": approval["mission_id"]}


# --- V2 PHASE 4 NOTIFICATION APIs ---
@app.get("/api/v1/notifications")
async def get_unacknowledged_notifications():
    """Fetch all unacknowledged system notifications."""
    return await colony_os.db.get_unacknowledged_notifications()


@app.post("/api/v1/notifications/{notification_id}/acknowledge")
async def acknowledge_notification(notification_id: str):
    """Acknowledge a system notification."""
    await colony_os.db.acknowledge_notification(notification_id)
    return {"status": "ACKNOWLEDGED", "notification_id": notification_id}


# --- V2 PHASE 4 MISSION INTERRUPTION & CONTROL APIs ---
@app.post("/api/v1/missions/{mission_id}/modify")
async def modify_running_mission(mission_id: str, req: ModifyMissionRequest):
    """Modify parameters of a running mission (topic, priority, subtask insertion)."""
    mission = await colony_os.db.get_mission(mission_id)
    if not mission:
        raise HTTPException(status_code=404, detail="Mission not found.")

    if req.new_topic:
        await colony_os.db.save_mission(mission_id, req.new_topic, MissionStatus(mission["status"]), DecisionStage(mission["stage"]))

    if req.insert_subtask:
        await colony_os.db.save_journal("Commander Prime", mission_id, f"[INTERRUPTION INSERT TASK]: {req.insert_subtask}")

    return {"status": "MODIFIED", "mission_id": mission_id}


if __name__ == "__main__":
    uvicorn.run("app:app", host="0.0.0.0", port=7860, reload=True)