app/core/scoring.py

"""
Scoring system for evaluating agent browsing interactions.

Five dimensions, each 0.0–1.0:
  - Completeness: Was the information found?
  - Confidence: How confident is the agent in its answer?
  - Efficiency: How few steps did it take relative to the max?
  - Speed: How fast was the run relative to a baseline?
  - Reliability: Did it complete successfully without errors?

Overall score is a weighted average of these dimensions.
"""

WEIGHTS = {
    "completeness": 0.30,
    "confidence": 0.25,
    "efficiency": 0.15,
    "speed": 0.10,
    "reliability": 0.20,
}

# Baseline: runs under this duration (seconds) get a perfect speed score
SPEED_BASELINE_SECONDS = 60.0


def compute_scores(
    found: bool,
    confidence: float,
    steps_taken: int,
    max_steps: int,
    duration_seconds: float,
    errors_encountered: int,
) -> dict:
    completeness = 1.0 if found else 0.0

    confidence_score = max(0.0, min(1.0, confidence))

    # Fewer steps = better. 1 step = 1.0, max_steps = 0.0
    if max_steps <= 1:
        efficiency = 1.0
    else:
        efficiency = max(0.0, 1.0 - (steps_taken - 1) / (max_steps - 1))

    # Faster = better. Under baseline = 1.0, scales down linearly to 0 at 5x baseline
    if duration_seconds <= SPEED_BASELINE_SECONDS:
        speed = 1.0
    else:
        speed = max(0.0, 1.0 - (duration_seconds - SPEED_BASELINE_SECONDS) / (4 * SPEED_BASELINE_SECONDS))

    # Base reliability from code errors: each error reduces by 0.25
    reliability = max(0.0, 1.0 - errors_encountered * 0.25)
    # If the task failed (not found), cap reliability at 0.5
    if not found:
        reliability = min(reliability, 0.5)

    scores = {
        "completeness": round(completeness, 3),
        "confidence": round(confidence_score, 3),
        "efficiency": round(efficiency, 3),
        "speed": round(speed, 3),
        "reliability": round(reliability, 3),
    }

    overall = sum(scores[k] * WEIGHTS[k] for k in WEIGHTS)
    scores["overall"] = round(overall, 3)

    return scores