| { |
| "name": "Evaluation regression", |
| "objective": "Detect regressions in agent behavior, connect them to recent prompt, context, or harness changes, and produce a verified repair proposal.", |
| "trigger": { |
| "type": "scheduled", |
| "cadence_or_event": "Nightly after eval completion, and when benchmark scores drop or trace graders fail." |
| }, |
| "intake": { |
| "sources": ["eval run results", "failing task IDs", "trace samples", "baseline scores", "recent prompt and harness commits"], |
| "selection_rule": "Investigate regression clusters with reproducible evidence; skip runs inside accepted variance or known-flaky sets." |
| }, |
| "workspace": { |
| "isolation": "Branch or sandbox with read access to traces and eval artifacts.", |
| "allowed_actions": ["targeted eval reruns", "scorer inspection", "small prompt, context, or fixture patches", "report generation"], |
| "disallowed_actions": ["scorer changes that hide failures", "benchmark cherry-picking", "broad prompt rewrites", "leaderboard claims"] |
| }, |
| "context": { |
| "required_files": ["evaluation rubric", "known-flaky eval list"], |
| "runtime_sources": ["baseline traces", "current traces", "model and runtime configuration"] |
| }, |
| "agents": [ |
| { |
| "role": "Investigator", |
| "responsibility": "Compare failing traces against passing baseline traces." |
| }, |
| { |
| "role": "Hypothesis writer", |
| "responsibility": "Classify the likely cause: model behavior, context, tool, scorer, fixture, or harness." |
| }, |
| { |
| "role": "Implementer", |
| "responsibility": "Patch the smallest plausible cause supported by trace evidence." |
| }, |
| { |
| "role": "Verifier", |
| "responsibility": "Rerun targeted evals, then a smoke suite, and check for new regressions." |
| }, |
| { |
| "role": "Judge", |
| "responsibility": "Decide whether evidence supports merging, deferring, or escalating." |
| } |
| ], |
| "verification": { |
| "gates": ["targeted failing tasks return to baseline", "no sentinel tasks regress", "trace evidence supports the claimed cause", "score deltas include run IDs and variance caveats"], |
| "receipts": ["eval run IDs", "trace excerpts", "hypotheses considered", "rerun scores"] |
| }, |
| "state": { |
| "artifacts": ["regression investigation notes", "patch-attempt log"], |
| "update_rule": "Record run IDs, failing tasks, hypotheses, patch attempts, rerun scores, and the final decision per regression cluster." |
| }, |
| "budget": { |
| "max_retries": 3, |
| "max_runtime_minutes": 120 |
| }, |
| "escalation": { |
| "conditions": ["scorer bug", "benchmark methodology change", "missing private traces", "model-provider incident", "fix risks overfitting"], |
| "destination": "Issue or PR tagged for the eval owner with reproducible evidence" |
| }, |
| "exit": { |
| "success": "The regression is repaired with verified reruns, or classified as flaky or scorer drift with evidence.", |
| "stop_without_success": "Artifacts are missing, retries are exhausted, or the tradeoff requires product judgment." |
| } |
| } |
|
|
