silent-failure-detector / src /agents /rule_based_agent.py
Harsh-9209's picture
fix(eval): mathematically bound all precision/recall metrics via laplace smoothing to naturally yield strict (0, 1) scores without artificial clipping
f8a6ce7
Raw
History Blame Contribute Delete
1.18 kB
"""Rule-based agent for the SilentFailureDetector environment.
This agent uses the pre-built heuristics from features.py:
- confidence marker count (certainty words β†’ higher risk)
- hedging marker count (hedging words β†’ lower risk)
- number density (many numbers β†’ higher risk of fabrication)
- combined simple_risk_score()
No LLM or training required β€” useful as a reproducible baseline.
"""
from src.features import simple_risk_score
from src.models import SilentFailureObservation
class RuleBasedAgent:
"""Deterministic heuristic agent.
Flags a response as risky when simple_risk_score() β‰₯ threshold.
The threshold is tunable; default (0.8) was chosen to maximise
F1 on the seed dataset's easy split.
"""
def __init__(self, threshold: float = 0.4) -> None:
self.threshold = threshold
def act(self, obs: SilentFailureObservation) -> int:
"""Return 1 (risky) or 0 (safe) based on the observation text."""
score = simple_risk_score(obs.text)
return 1 if score >= self.threshold else 0
def __repr__(self) -> str:
return f"RuleBasedAgent(threshold={self.threshold})"