Spaces:
Sleeping
Sleeping
File size: 4,064 Bytes
98ee05e e6fb135 98ee05e e6fb135 98ee05e e6fb135 98ee05e e6fb135 98ee05e e6fb135 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 | """Eval runner: re-scores the golden dataset against a prompt version.
For each golden entry, regenerate copy for the same brief + variant, re-judge
it, and compare to the stored baseline. A drop > REGRESSION_TOLERANCE (0.5)
fails the run. Guards prompt-version swaps and backs the pytest regression suite.
"""
from dataclasses import dataclass, field
from typing import Any, Optional
from agent import prompts, tools
from evals.golden import GoldenDataset
from evals.judge import judge_output
# A golden entry may not drop more than this from its baseline before we fail.
REGRESSION_TOLERANCE = 0.5
@dataclass
class EntryResult:
brief: dict[str, Any]
variant_type: str
baseline_score: float
new_score: float
new_output: str
regressed: bool
@property
def delta(self) -> float:
return round(self.new_score - self.baseline_score, 4)
@dataclass
class EvalReport:
prompt_version: str
results: list[EntryResult] = field(default_factory=list)
@property
def passed(self) -> bool:
return not any(r.regressed for r in self.results)
@property
def regressions(self) -> list[EntryResult]:
return [r for r in self.results if r.regressed]
@property
def count(self) -> int:
return len(self.results)
def _generate_variant(brief: dict[str, Any], variant_type: str, prompt_version: str) -> str:
"""Regenerate a single variant for a brief using a given prompt version.
Few-shot examples are intentionally omitted so the regression isolates the
prompt itself rather than whatever happens to be in memory.
"""
prompt = prompts.render_generation_prompt(brief, few_shot_block="", version=prompt_version)
raw = tools.chat(prompt, temperature=0.7)
parsed = tools.extract_json(raw)
return str(parsed.get(variant_type, "")).strip()
def run_golden_eval(
prompt_version: Optional[str] = None,
golden: Optional[GoldenDataset] = None,
) -> EvalReport:
"""Run every golden entry against `prompt_version` (defaults to active)."""
prompt_version = prompt_version or prompts.ACTIVE_PROMPT_VERSION
golden = golden or GoldenDataset()
report = EvalReport(prompt_version=prompt_version)
for entry in golden.all():
brief = entry["brief"]
variant_type = entry["variant_type"]
baseline = float(entry["weighted_average"])
new_output = _generate_variant(brief, variant_type, prompt_version)
new_scores = judge_output(brief, variant_type, new_output)
new_score = float(new_scores["weighted_average"])
regressed = (baseline - new_score) > REGRESSION_TOLERANCE
report.results.append(
EntryResult(
brief=brief,
variant_type=variant_type,
baseline_score=baseline,
new_score=new_score,
new_output=new_output,
regressed=regressed,
)
)
return report
def format_report(report: EvalReport) -> str:
"""Plain-text summary of a regression run (used as a fallback to Rich)."""
lines = [
f"Regression eval for prompt version: {report.prompt_version}",
f"Entries checked: {report.count}",
f"Tolerance: drop > {REGRESSION_TOLERANCE} fails",
"",
]
if report.count == 0:
lines.append("No golden entries yet - nothing to check. (PASS)")
return "\n".join(lines)
for i, r in enumerate(report.results, 1):
status = "REGRESSED" if r.regressed else "ok"
lines.append(
f" [{i}] {r.variant_type:8s} baseline={r.baseline_score:.2f} "
f"new={r.new_score:.2f} delta={r.delta:+.2f} {status}"
)
lines.append("")
if report.passed:
lines.append("RESULT: PASS - no entry regressed beyond tolerance.")
else:
lines.append(
f"RESULT: FAIL - {len(report.regressions)} entry(ies) regressed beyond "
f"{REGRESSION_TOLERANCE}. Do not promote this prompt version."
)
return "\n".join(lines)
|