Spaces:
Sleeping
Sleeping
File size: 2,260 Bytes
98ee05e e6fb135 98ee05e e6fb135 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 | """Runs after each generation: promote winners into the golden dataset and
ChromaDB memory (>= GOLDEN_THRESHOLD), flag the weak ones (< FLAG_THRESHOLD).
This closes the loop - promoted outputs become future few-shot examples.
"""
from datetime import datetime, timezone
from typing import Any
from agent.memory import Memory
from db.store import Store
from evals.golden import GOLDEN_THRESHOLD, GoldenDataset
# Outputs scoring strictly below this are flagged for human review.
FLAG_THRESHOLD = 2.5
FLAG_REASON = "below quality threshold"
def run_feedback(
store: Store,
memory: Memory,
brief: dict[str, Any],
run_id: int,
scored_outputs: list[dict[str, Any]],
prompt_version: str,
) -> dict[str, Any]:
"""Process one run's outputs: promote the good, flag the bad.
`scored_outputs` is a list of {variant_type, content, scores} dicts.
Returns a summary describing what changed.
"""
golden = GoldenDataset(store=store)
now = datetime.now(timezone.utc).isoformat()
promoted: list[str] = []
flagged: list[str] = []
for item in scored_outputs:
variant_type = item["variant_type"]
content = item["content"]
scores = item["scores"]
weighted = float(scores.get("weighted_average", 0.0))
if weighted >= GOLDEN_THRESHOLD:
added = golden.maybe_add(brief, variant_type, content, scores, prompt_version)
memory.add(
brief=brief,
variant_type=variant_type,
output=content,
score=weighted,
prompt_version=prompt_version,
timestamp=now,
)
if added:
promoted.append(variant_type)
elif weighted < FLAG_THRESHOLD:
store.add_flagged(
brief=brief,
variant_type=variant_type,
output=content,
weighted_average=weighted,
reason=FLAG_REASON,
run_id=run_id,
)
flagged.append(variant_type)
return {
"promoted_to_golden": promoted,
"flagged_for_review": flagged,
"golden_threshold": GOLDEN_THRESHOLD,
"flag_threshold": FLAG_THRESHOLD,
}
|