feedback/loop.py

"""Runs after each generation: promote winners into the golden dataset and
ChromaDB memory (>= GOLDEN_THRESHOLD), flag the weak ones (< FLAG_THRESHOLD).

This closes the loop - promoted outputs become future few-shot examples.
"""

from datetime import datetime, timezone
from typing import Any

from agent.memory import Memory
from db.store import Store
from evals.golden import GOLDEN_THRESHOLD, GoldenDataset

# Outputs scoring strictly below this are flagged for human review.
FLAG_THRESHOLD = 2.5
FLAG_REASON = "below quality threshold"


def run_feedback(
    store: Store,
    memory: Memory,
    brief: dict[str, Any],
    run_id: int,
    scored_outputs: list[dict[str, Any]],
    prompt_version: str,
) -> dict[str, Any]:
    """Process one run's outputs: promote the good, flag the bad.

    `scored_outputs` is a list of {variant_type, content, scores} dicts.
    Returns a summary describing what changed.
    """
    golden = GoldenDataset(store=store)
    now = datetime.now(timezone.utc).isoformat()

    promoted: list[str] = []
    flagged: list[str] = []

    for item in scored_outputs:
        variant_type = item["variant_type"]
        content = item["content"]
        scores = item["scores"]
        weighted = float(scores.get("weighted_average", 0.0))

        if weighted >= GOLDEN_THRESHOLD:
            added = golden.maybe_add(brief, variant_type, content, scores, prompt_version)
            memory.add(
                brief=brief,
                variant_type=variant_type,
                output=content,
                score=weighted,
                prompt_version=prompt_version,
                timestamp=now,
            )
            if added:
                promoted.append(variant_type)

        elif weighted < FLAG_THRESHOLD:
            store.add_flagged(
                brief=brief,
                variant_type=variant_type,
                output=content,
                weighted_average=weighted,
                reason=FLAG_REASON,
                run_id=run_id,
            )
            flagged.append(variant_type)

    return {
        "promoted_to_golden": promoted,
        "flagged_for_review": flagged,
        "golden_threshold": GOLDEN_THRESHOLD,
        "flag_threshold": FLAG_THRESHOLD,
    }