evaluation.py

"""
Outcome Evaluator — self-judgment plugin (Success / Failure path).

Implements the self-assessment loop described in the spec:
  - Validates actual output against expected outcome
  - Routes to Success → distill procedural strategy
  - Routes to Failure → extract negative guardrail + run reflection
  - Feeds result back into SAFLA consolidation
"""
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional

from src.memory.models import MemoryOutcome, ExperientialStrategy
from src.memory.experiential import ExperientialRepository
from src.memory.safla import update_confidence


@dataclass
class EvaluationResult:
    outcome:          MemoryOutcome
    score:            float = 0.0          # 0–1 quality estimate
    reflection:       str  = ""
    new_strategy_id:  Optional[str] = None
    new_guardrail_ids: List[str] = field(default_factory=list)


class OutcomeEvaluator:
    """
    Self-judgment component of the SAFLA loop.

    Flow:
        Agent Output → judge() → EvaluationResult
                                   ├── SUCCESS → distill_strategy()
                                   └── FAILURE → extract_guardrail() via ExperientialRepository
    """

    def __init__(self, repository: Optional[ExperientialRepository] = None):
        self._repo = repository or ExperientialRepository()

    def judge(
        self,
        objective: str,
        output: Any,
        expected_keywords: Optional[List[str]] = None,
        error: Optional[str] = None,
    ) -> EvaluationResult:
        """Determine success/failure and score the output."""
        if error:
            return EvaluationResult(
                outcome=MemoryOutcome.FAILURE,
                score=0.0,
                reflection=f"Execution error: {error}",
            )
        if output is None:
            return EvaluationResult(
                outcome=MemoryOutcome.FAILURE,
                score=0.0,
                reflection="No output produced.",
            )
        # Score based on keyword overlap if hints are provided
        score = 1.0
        reflection = "Execution completed successfully."
        if expected_keywords:
            output_text = str(output).lower()
            matched = sum(1 for kw in expected_keywords if kw.lower() in output_text)
            score = matched / len(expected_keywords)
            if score < 0.5:
                return EvaluationResult(
                    outcome=MemoryOutcome.FAILURE,
                    score=score,
                    reflection=f"Output matched only {matched}/{len(expected_keywords)} expected keywords.",
                )
            reflection = f"Output matched {matched}/{len(expected_keywords)} expected keywords."
        return EvaluationResult(
            outcome=MemoryOutcome.SUCCESS,
            score=score,
            reflection=reflection,
        )

    def consolidate(
        self,
        eval_result: EvaluationResult,
        objective: str,
        output_summary: str,
        used_strategy_ids: Optional[List[str]] = None,
    ) -> EvaluationResult:
        """
        Post-judgment SAFLA consolidation step.
        Distills a new strategy or guardrail and updates confidence
        on any strategies used during this task.
        """
        if eval_result.outcome == MemoryOutcome.SUCCESS:
            strategy = self._repo.distill_from_outcome(
                title=f"Strategy: {objective[:60]}",
                description=objective,
                content=output_summary,
                outcome=MemoryOutcome.SUCCESS,
                task_pattern=objective,
            )
            eval_result.new_strategy_id = strategy.item_id
        else:
            # Strip user-specific params → evergreen guardrail
            guardrail = self._repo.distill_from_outcome(
                title=f"Avoid: {objective[:60]}",
                description=eval_result.reflection,
                content=f"Pattern to avoid: {objective}. Reason: {eval_result.reflection}",
                outcome=MemoryOutcome.FAILURE,
                task_pattern=objective,
            )
            eval_result.new_guardrail_ids.append(guardrail.item_id)

        # Apply SAFLA to all strategies used in this task
        if used_strategy_ids:
            new_guardrails = self._repo.record_outcome(
                used_strategy_ids, eval_result.outcome
            )
            eval_result.new_guardrail_ids.extend(new_guardrails)

        return eval_result