lib/affect/refusal_detector.py

"""
Value-based refusal detector for agent text.

Three layers, text-only — no LLM required:

1. Morphology (pymorphy3): normalized forms of refusal verbs
   and constructions like "negation + modal verb".
2. Semantic value context (NRC EmoLex): high density of disgust or anger
   signals moral/ethical context, distinguishing "I won't participate in
   deception" from "no, that's incorrect".
3. Capability exclusion: if a refusal verb is near a technical action verb,
   it's "I can't generate" rather than "I won't cause harm".

Optional LLM-fallback connects via `RefusalDetectorConfig` and is used only
when both layers don't give a confident answer. If LLM is not configured,
the system stays silent.
"""

from __future__ import annotations

import math
import re
from collections.abc import Callable
from dataclasses import dataclass

from lib.affect.emolex.emolex import _lemma_ru_cached as lemmatize
from lib.affect.emolex.emolex import emotion_score, tokenize

# ---------------------------------------------------------------------------
# Normalized forms — value refusal vocabulary
# ---------------------------------------------------------------------------

# Verbs that are inherently refusals
_REFUSAL_VERB_NORMALS = frozenset(
    [
        # Russian
        "отказываться",
        "отказаться",
        "отклонить",
        "отклонять",
        "воздержаться",
        "воздерживаться",
        "отвергнуть",
        "отвергать",
        # English (pymorphy3 passes latin as-is, lowercase)
        "refuse",
        "decline",
        "reject",
        "abstain",
    ]
)

# Verbs that become refusal through negation: "won't", "will not", "don't want"
_MODAL_NEGATABLE = frozenset(
    [
        # Russian
        "мочь",  # не могу
        "стать",  # не стану
        "хотеть",  # не хочу
        "собираться",  # не собираюсь
        "помогать",  # не буду помогать
        "помочь",
        "делать",
        "участвовать",
        # English
        "will",  # will not / won't
        "can",  # cannot / can't
        "going",  # not going to
        "participate",
        "assist",
        "help",
    ]
)

# Technical inability context → NOT a value refusal
_CAPABILITY_NORMALS = frozenset(
    [
        "генерировать",
        "рисовать",
        # create/build — capability, but only if no moral context
        "создавать",
        "создать",
        # run/launch — both aspects
        "запускать",
        "запустить",
        # install/download
        "установить",
        "устанавливать",
        "загружать",
        "загрузить",
        # open/connect/get
        "открывать",
        "открыть",
        "подключаться",
        "подключиться",
        "получить",
        "получать",
        # perceive
        "слышать",
        "видеть",
        "читать",
        "уметь",  # «не умею» — capability
        "знать",  # «не знаю» — knowledge gap
        "иметь",  # «не имею возможности» — capability
        "выполнить",
        "выполнять",  # «не могу выполнить» — technical
        "обработать",
        "обрабатывать",
        # English capability verbs
        "generate",
        "draw",
        "create",
        "run",
        "execute",
        "install",
        "download",
        "upload",
        "open",
        "connect",
        "read",
        "write",
        "hear",
        "see",
        "know",  # "I don't know" — knowledge gap
        "access",
        "process",
    ]
)

# Explicit refusal phrases — when present, count as a refusal signal even
# without a canonical refusal verb or negated modal. Used to catch concise
# value-driven refusals ("Поэтому — нет", "So — no") that rephrase the
# refusal but don't use the standard morphological forms.
_EXPLICIT_REFUSAL_PHRASES = (
    # Russian
    "поэтому — нет",
    "поэтому нет",
    "мой ответ — нет",
    "мой ответ нет",
    "ответ — нет",
    "ответ нет",
    "и нет",  # "...и нет." as terse refusal closer
    # English
    "so — no",
    "so, no",
    "so no.",
    "the answer is no",
    "my answer is no",
)


# Negation markers that are valid as scope indicators
_NEGATORS = frozenset(
    [
        # Russian
        "не",
        "ни",
        "нельзя",
        "невозможно",
        "нет",
        # English
        "not",
        "never",
        "cannot",
    ]
)

# Stems of English contractions whose negation is split by the apostrophe-stripping
# tokenizer. "won't" → ["won", "t"], "don't" → ["don", "t"], etc. The detector
# treats the pair (stem, "t") as a single negator.
_CONTRACTION_NEG_STEMS = frozenset(
    [
        "won",
        "don",
        "doesn",
        "didn",
        "isn",
        "aren",
        "wasn",
        "weren",
        "haven",
        "hasn",
        "hadn",
        "shouldn",
        "couldn",
        "wouldn",
        "ain",
        "mustn",
        "needn",
        "can",  # "can't" → ["can", "t"]; bare "can" never reaches this list because
        # it never appears together with a trailing "t" outside contractions.
    ]
)

# Narrower subset: contractions where the modal expresses refusal of intent,
# inability, or moral disagreement. Excludes copula ("isn't", "aren't") and
# aspect ("haven't") contractions — those negate state/perfect, not action.
# Used by score_refusal Pattern B to avoid firing on neutral "isn't about X".
_MODAL_REFUSAL_CONTRACTION_STEMS = frozenset(
    [
        "won",     # won't = will not — intent refusal
        "can",     # can't = cannot — inability/refusal
        "shouldn", # shouldn't — moral refusal
        "wouldn",  # wouldn't — preference refusal
        "couldn",  # couldn't — inability
        "don",     # don't = do not — action refusal
    ]
)

# Value-context phrases — when present in the same text as a refusal marker,
# they signal ethical/moral framing even when NRC density alone is too low
# (common for short or hedged English refusals). Used as a fallback to
# unblock `no_moral_context` cases that are genuinely value-driven.
#
# Only multi-word phrases tied to ethical/refusal context are included.
# Bare "against my" / "violates my" / "compromise my" / "not willing to"
# were considered but dropped (false positives in technical text:
# "works against my expectations", "violates my linter rules", etc).
_VALUE_CONTEXT_PHRASES = (
    # ── Russian ──────────────────────────────────────────────────────────
    "против моих принципов",
    "против моих ценностей",
    "моих принципов",
    "моих ценностей",
    "это неправильно",
    "недопустимо",
    "причинит вред",
    "навредит человеку",
    "навредит людям",
    "обманывать",
    "обмануть человека",
    "обмануть людей",
    "врать пользователю",
    "лгать пользователю",
    "предать доверие",
    "предам доверие",
    "пересекает черту",
    "пересечь черту",
    "против совести",
    # ── English ──────────────────────────────────────────────────────────
    "against what i",
    "against what i believe",
    "against my principles",
    "against my values",
    "harm someone",
    "harm them",
    "would harm",
    "could harm",
    "betray",
    "betraying",
    "morally wrong",
    "my principles",
    "my values",
    "crosses a line",
    "cross a line",
    "deceive",
    "deceiving",
    "dishonest",
    "going against my",
    "who i want to be",
    "wrong to do",
)


def _window_has_negation(lemmas: list[str], end: int, size: int) -> bool:
    """True when the window of `size` lemmas ending at `end` contains a negator.

    Recognises both bare negators ("not", "не") and the contraction-pair pattern
    where the tokenizer split off the apostrophe (e.g. ["won", "t"]).
    """
    start = max(0, end - size)
    window = lemmas[start:end]
    if any(w in _NEGATORS for w in window):
        return True
    for j in range(len(window) - 1):
        if window[j + 1] == "t" and window[j] in _CONTRACTION_NEG_STEMS:
            return True
    return False


def _capability_context_near(lemmas: list[str], i: int, radius: int = 5) -> bool:
    """True iff a capability verb lemma is within ±radius tokens of position i.

    Used to distinguish "I can't generate images" (capability — discount)
    from "I refuse to deceive. I can see your point" (value refusal, with
    an unrelated capability verb later in the message).
    """
    start = max(0, i - radius)
    end = min(len(lemmas), i + radius + 1)
    return any(lem in _CAPABILITY_NORMALS for lem in lemmas[start:end])


# NRC threshold (density per 100 tokens) for "moral context".
# EN was 2.0 — too low; almost any negatively-charged token cleared it,
# triggering false positives on neutral text discussing risk/concern.
_MORAL_THRESHOLD_RU = 8.0
_MORAL_THRESHOLD_EN = 4.0


# ---------------------------------------------------------------------------
# Configuration
# ---------------------------------------------------------------------------


@dataclass
class RefusalDetectorConfig:
    """
    Value refusal detector configuration.

    Without LLM (default):
      Only text layers operate.
      confidence < uncertain_threshold → record nothing.

    With LLM (optional):
      If confidence is in uncertain zone AND llm_classifier is set,
      the classifier is invoked. It makes the final decision.
    """

    # Threshold below which we stay silent (avoid false positives)
    min_confidence: float = 0.45

    # Uncertain zone — delegate to LLM if configured
    uncertain_low: float = 0.45
    uncertain_high: float = 0.65

    # Optional LLM classifier (sync: text -> bool)
    # Signature: (text: str) -> bool
    # If None, LLM is never used.
    llm_classifier: Callable[[str], bool] | None = None


# ---------------------------------------------------------------------------
# Core logic
# ---------------------------------------------------------------------------


@dataclass
class RefusalScore:
    confidence: float  # 0.0–1.0
    has_refusal_verb: bool
    has_negated_modal: bool
    has_capability_context: bool
    disgust_density: float
    anger_density: float
    decided_by: str  # "text", "llm", "below_threshold"

    @property
    def is_value_refusal(self) -> bool:
        # Mirrors RefusalDetectorConfig.min_confidence default. Callers that
        # need a different threshold should pass a config to is_value_refusal().
        return self.confidence >= 0.45


def score_refusal(text: str) -> RefusalScore:
    """
    Compute the degree of value refusal in text.

    Returns RefusalScore with confidence 0.0–1.0.
    Confidence interpretation:
      < 0.45  — not a refusal (or uncertain)
      0.45–0.65 — "gray zone" (LLM could help)
      > 0.65  — value refusal with high confidence

    Internal formula:
      confidence = refusal_signal * moral_signal * (1 - capability_discount)

    refusal_signal: 1.0 if refusal verb or negated_modal present
    moral_signal: depends on disgust/anger density in NRC
    capability_discount: 0.85 if capability verb is nearby
    """
    # Strip think-blocks before analysis
    clean = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL)

    tokens = tokenize(clean)
    if not tokens:
        return RefusalScore(0.0, False, False, False, 0.0, 0.0, "below_threshold")

    lemmas = [lemmatize(t) for t in tokens]
    lemma_set = set(lemmas)

    # ── Layer 1: morphology ──────────────────────────────────────────────
    refusal_position: int | None = None
    has_refusal_verb = False
    has_negated_modal = False

    # Pattern C — explicit terse refusal phrase anywhere in the text.
    # Treated as equivalent to a refusal verb for the signal layer.
    clean_lower = clean.lower()
    if any(phrase in clean_lower for phrase in _EXPLICIT_REFUSAL_PHRASES):
        has_refusal_verb = True
        refusal_position = 0  # whole-text marker — position irrelevant

    for i, lemma in enumerate(lemmas):
        if lemma in _REFUSAL_VERB_NORMALS:
            has_refusal_verb = True
            refusal_position = i
            break

    if refusal_position is None:
        for i, lemma in enumerate(lemmas):
            # Pattern A — bare modal preceded by negation in window
            if lemma in _MODAL_NEGATABLE and _window_has_negation(lemmas, i, 3):
                has_negated_modal = True
                refusal_position = i
                break
            # Pattern B — English modal-refusal contraction at position i + "t"
            # at i+1. Only true refusal modals (won/can/shouldn/wouldn/couldn/don),
            # NOT copula contractions (isn/aren/wasn) which negate state.
            if (
                lemma in _MODAL_REFUSAL_CONTRACTION_STEMS
                and i + 1 < len(lemmas)
                and lemmas[i + 1] == "t"
            ):
                has_negated_modal = True
                refusal_position = i
                break

    # Capability context: only counts if it's near the refusal verb (±5 tokens),
    # not anywhere in the text. Previously a global set-intersection turned
    # any mention of "see"/"read"/"know" into a global discount.
    if refusal_position is not None:
        has_capability_context = _capability_context_near(lemmas, refusal_position, radius=5)
    else:
        has_capability_context = bool(_CAPABILITY_NORMALS & lemma_set)

    refusal_signal = 1.0 if (has_refusal_verb or has_negated_modal) else 0.0

    if math.isclose(refusal_signal, 0.0):
        return RefusalScore(
            0.0, False, False, has_capability_context, 0.0, 0.0, "no_refusal"
        )

    # ── Layer 2: NRC moral context ───────────────────────────────────────
    lang = "ru" if _is_mostly_cyrillic(clean) else "en"
    try:
        scores = emotion_score(clean, lang=lang)
    except Exception:
        scores = {}

    disgust = float(scores.get("disgust", 0.0))
    anger = float(scores.get("anger", 0.0))
    fear = float(scores.get("fear", 0.0))
    # Moral signal: disgust is specific to norm violations,
    # anger appears with injustice; disgust weighs more.
    # For English text, fear also carries ethical weight
    # (harm, danger, deception), while capability refusals have fear=0.
    is_ru = _is_mostly_cyrillic(clean)
    if is_ru:
        moral_density = disgust + anger * 0.5
        moral_threshold = _MORAL_THRESHOLD_RU
    else:
        moral_density = disgust + anger * 0.5 + fear * 0.35
        moral_threshold = _MORAL_THRESHOLD_EN

    # Normalize to [0, 1]. Divisor differs: English NRC yields lower densities.
    divisor = 15.0 if is_ru else 5.0
    moral_signal = min(1.0, moral_density / divisor) if moral_density >= moral_threshold else 0.0

    if math.isclose(moral_signal, 0.0):
        # NRC density too low for moral context — but the agent may still be
        # value-refusing in English where moral terms are often abstract
        # ("against my values", "would harm", "betray", "crosses a line")
        # and don't have direct NRC entries. Fallback: scan for explicit
        # value-context phrases. Short refusals like "I won't do this. It's
        # not a rule — it would harm someone" are caught here.
        lower_text = clean.lower()
        value_hits = sum(1 for p in _VALUE_CONTEXT_PHRASES if p in lower_text)
        if value_hits >= 1:
            moral_signal = 0.6 if value_hits == 1 else 0.85
        else:
            # No refusal-supporting context at all — gray zone for LLM fallback.
            confidence = 0.30
            return RefusalScore(
                confidence,
                has_refusal_verb,
                has_negated_modal,
                has_capability_context,
                disgust,
                anger,
                "no_moral_context",
            )

    # ── Layer 3: technical inability discount ────────────────────────────
    # Capability discount is now soft (0.5 not 0.85): if a refusal verb sits
    # next to a capability verb ("I cannot generate images"), we still keep
    # half the moral confidence — moral content + capability context isn't
    # rare. Old 0.85 effectively vetoed any refusal that mentioned a
    # capability verb anywhere in the message.
    capability_discount = 0.5 if has_capability_context else 0.0
    confidence = refusal_signal * (0.4 + 0.6 * moral_signal) * (1.0 - capability_discount)

    return RefusalScore(
        confidence=round(confidence, 3),
        has_refusal_verb=has_refusal_verb,
        has_negated_modal=has_negated_modal,
        has_capability_context=has_capability_context,
        disgust_density=disgust,
        anger_density=anger,
        decided_by="text",
    )


def is_value_refusal(
    text: str,
    config: RefusalDetectorConfig | None = None,
) -> bool:
    """
    Main entry point: True if text contains a value refusal.

    Without LLM: decision by text only.
    With LLM (config.llm_classifier set): invoked in uncertain zone.
    """
    cfg = config or RefusalDetectorConfig()
    result = score_refusal(text)

    if result.confidence < cfg.min_confidence:
        return False

    if (
        cfg.llm_classifier is not None
        and cfg.uncertain_low <= result.confidence <= cfg.uncertain_high
    ):
        try:
            return cfg.llm_classifier(text)
        except Exception:  # nosec B110
            pass  # LLM unavailable — decide by text only

    return result.confidence >= cfg.min_confidence


def _is_mostly_cyrillic(text: str) -> bool:
    sample = text[:200]
    cyr = sum(1 for c in sample if "Ѐ" <= c <= "ӿ")
    lat = sum(1 for c in sample if "a" <= c.lower() <= "z")
    return cyr >= lat