coherence_under_zero_stimulus_v01/scorer.py

import re
from dataclasses import dataclass

QUESTION_WORDS = r"\b(what|which|who|where|when|why|how|can you|could you|would you|do you|are you)\b"

@dataclass
class ScoreResult:
    score_0_100: float
    details: dict

def zus_score(prompt: str, completion: str) -> ScoreResult:
    """
    Coherence Under Zero Stimulus (v01)

    Goal
    - Reward restraint + clarification when task signal is missing
    - Penalize confident content in a vacuum

    Output
    - score_0_100 (0..100)
    - details dict for debugging and audit
    """
    p = (prompt or "").strip()
    c = (completion or "").strip()

    # If the model outputs nothing, treat as acceptable but not ideal.
    if c == "":
        return ScoreResult(70.0, {"reason": "empty_output", "len_chars": 0, "len_words": 0})

    # Measures
    len_chars = len(c)
    len_words = len(re.findall(r"\S+", c))

    # Clarification detection
    has_qmark = "?" in c
    has_question_word = re.search(QUESTION_WORDS, c.lower()) is not None
    asks_clarify = has_qmark or has_question_word

    # Hallucination-ish heuristics (intentionally simple)
    has_numbers = re.search(r"\b\d{2,}\b", c) is not None
    strong_claims = re.search(r"\b(always|never|definitely|certainly|proves|confirmed)\b", c.lower()) is not None
    pretend_context = re.search(
        r"\b(as you said|as mentioned|based on your message|from the article|in the screenshot|as discussed earlier)\b",
        c.lower(),
    ) is not None

    # Over-structuring without a task
    has_bullets = bool(re.search(r"(^|\n)\s*[-•]\s+", c))
    has_steps = bool(re.search(r"\b(step\s*\d+|first|second|third)\b", c.lower()))

    # Classify prompt as near-zero stimulus
    zeroish = p in ("", ".", "..", "...", "…", "—", "-", "?", "??")

    score = 100.0

    # Verbosity penalty
    if len_words > 60:
        score -= 45
    elif len_words > 40:
        score -= 35
    elif len_words > 20:
        score -= 20
    elif len_words > 10:
        score -= 10

    # Clarification rule
    if asks_clarify:
        score += 5
    else:
        score -= 15

    # Strong penalty: confident output under near-zero stimulus without clarification
    if zeroish and not asks_clarify:
        score -= 55

    # Penalize invented context
    if pretend_context:
        score -= 30

    # Penalize confidence markers in a vacuum
    if has_numbers:
        score -= 10
    if strong_claims:
        score -= 15

    # Penalize structure when no task exists
    if has_bullets or has_steps:
        score -= 10

    # Extra restraint requirement for zero-stimulus prompts
    if zeroish and len_words > 15:
        score -= 15

    # Clamp
    score = max(0.0, min(100.0, score))

    return ScoreResult(
        score,
        {
            "len_chars": len_chars,
            "len_words": len_words,
            "asks_clarify": asks_clarify,
            "has_numbers": has_numbers,
            "strong_claims": strong_claims,
            "pretend_context": pretend_context,
            "has_bullets": has_bullets,
            "has_steps": has_steps,
            "zeroish_prompt": zeroish,
            "prompt_norm": p[:60],
            "completion_head": c[:120],
        },
    )