app/pipeline/candidate_validator.py

"""Validate rewrite candidates for meaning, polarity, and fact preservation."""

from __future__ import annotations

import re
from dataclasses import dataclass

from app.pipeline.meaning_safety import polarity_safe

# Tokens that look like proper nouns / identifiers to preserve.
_PROPER = re.compile(r"\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)*)\b")
_NUMBER = re.compile(
    r"\b(?:\d{1,3}(?:,\d{3})+(?:\.\d+)?|\d+\.\d+%?|\d+%?|\d{4})\b"
)
# Straight/curly double quotes only — apostrophes in don't/it's are not quotes.
_QUOTED = re.compile(r"[\"“”]([^\"“”]{2,120})[\"“”]")
_YEAR = re.compile(r"\b(?:19|20)\d{2}\b")
_SENT_START = re.compile(r"(?:^|[.!?]\s+)([A-Z])")


@dataclass
class ValidationResult:
    ok: bool
    reasons: list[str]
    meaning: float = 1.0
    surface_sim: float = 1.0


def _surface_sim(a: str, b: str) -> float:
    from difflib import SequenceMatcher

    return SequenceMatcher(None, a.lower().strip(), b.lower().strip()).ratio()


def _extract_numbers(text: str) -> set[str]:
    return {m.group(0) for m in _NUMBER.finditer(text)} | {
        m.group(0) for m in _YEAR.finditer(text)
    }


def _extract_quotes(text: str) -> set[str]:
    return {m.group(1).strip().lower() for m in _QUOTED.finditer(text)}


def _extract_propers(text: str) -> set[str]:
    """Capitalized multi-word / mid-sentence names (skip sentence starts)."""
    found: set[str] = set()
    # Mark character offsets that are sentence-initial capitals
    sentence_starts = {m.start(1) for m in _SENT_START.finditer(text)}
    for m in _PROPER.finditer(text):
        start = m.start()
        if start in sentence_starts:
            continue
        if start == 0:
            continue
        # Also skip after whitespace that follows a sentence boundary
        i = start - 1
        while i >= 0 and text[i].isspace():
            i -= 1
        if i < 0 or text[i] in ".!?":
            continue
        found.add(m.group(1))
    return found


_STOP = frozenset(
    """
    a an the and or but if in on at to for of as by with from into over after
    before about than then so too very more most such that this these those
    it its they them their we our you your he she his her is are was were be
    been being have has had do does did will would can could should may might
    not no nor also just only own same other another each every both few many
    much some any all because while when where how what which who whom
    """.split()
)


def _light_stem(token: str) -> str:
    """Shared light stem so topic/coverage checks agree on the same forms."""
    t = (token or "").strip("'").lower()
    if t.endswith("ing") and len(t) > 5:
        return t[:-3]
    if t.endswith("ed") and len(t) > 4:
        return t[:-2]
    if t.endswith("ies") and len(t) > 4:
        return t[:-3] + "y"
    if t.endswith("es") and len(t) > 4:
        return t[:-2]
    if t.endswith("s") and len(t) > 4:
        return t[:-1]
    return t


def _content_tokens(text: str) -> set[str]:
    """Lowercased content words (length ≥4) used for coverage checks."""
    toks = re.findall(r"[a-zA-Z']+", text.lower())
    out: set[str] = set()
    for t in toks:
        t = t.strip("'")
        if len(t) < 4 or t in _STOP:
            continue
        out.add(_light_stem(t))
    return out


def _sentence_count(text: str) -> int:
    parts = re.split(r"[.!?]+", (text or "").strip())
    return len([p for p in parts if p.strip()])


def _unit_shape_ok(original: str, candidate: str) -> bool:
    """Single-sentence units must stay one sentence; never shrink claims away."""
    o_sents = _sentence_count(original)
    c_sents = _sentence_count(candidate)
    # One in → one out (SmolLM often appends a second invented sentence)
    if o_sents == 1 and c_sents != 1:
        return False
    if o_sents >= 2 and c_sents < o_sents:
        return False
    ow = max(1, len(original.split()))
    cw = len(candidate.split())
    # Reject heavy truncation of a single claim
    if ow <= 40 and cw < max(4, int(ow * 0.78)):
        return False
    return True


def _length_ok(original: str, candidate: str) -> bool:
    ow = max(1, len(original.split()))
    cw = len(candidate.split())
    # Reject half-dropped paraphrases (FLAN/Smol often keep only the last sentence).
    if ow <= 80:
        min_words = max(3, int(ow * 0.78))
    else:
        min_words = max(3, int(ow * 0.55))
    if cw < min_words:
        return False
    if cw > int(ow * 2.0) + 6:
        return False
    return True


def _coverage_ok(original: str, candidate: str) -> bool:
    """Reject dropped sentences / severe content loss; allow normal paraphrases."""
    o_sents = _sentence_count(original)
    c_sents = _sentence_count(candidate)
    # Short multi-sentence passages must keep every sentence (anti half-drop).
    if 2 <= o_sents <= 5 and c_sents < o_sents:
        return False

    o_toks = _content_tokens(original)
    if len(o_toks) >= 5:
        c_toks = _content_tokens(candidate)
        overlap = len(o_toks & c_toks) / max(1, len(o_toks))
        # Tighter than before: FLAN often keeps topic words while dropping claims.
        if overlap < 0.48:
            return False
    return True


def _key_stems(text: str) -> set[str]:
    """Claim-anchor stems: all content stems with len≥4 (book, city, save, …)."""
    return {t for t in _content_tokens(text) if len(t) >= 4}


def _key_content_ok(original: str, candidate: str) -> bool:
    """Require most distinctive source stems to survive the rewrite.

    Catches 'Online learning' → 'Learning', 'Reading books' → 'Reading',
    and '…in big cities' drops.
    """
    keys = _key_stems(original)
    # Short units: use every content stem so short nouns are not skipped.
    if len(original.split()) <= 28 or len(keys) < 3:
        keys = _content_tokens(original) or keys
    if len(keys) < 2:
        return True
    c_toks = _content_tokens(candidate)
    kept = len(keys & c_toks) / len(keys)
    ow = len(original.split())
    need = 0.82 if ow <= 28 else 0.72
    return kept >= need


def _ordered_content_stems(text: str) -> list[str]:
    """Content stems in left-to-right order (for topic-head checks)."""
    toks = re.findall(r"[a-zA-Z']+", (text or "").lower())
    out: list[str] = []
    for t in toks:
        t = t.strip("'")
        if len(t) < 4 or t in _STOP:
            continue
        out.append(_light_stem(t))
    return out


_WEAK_TOPIC_HEADS = frozenset(
    """
    public private many some more most other another every often usual common
    local young loud hard free quiet small large great important different
    second first major main recent modern social early late daily
    without within enough several various certain further additional
    unfortunately sadly however therefore moreover furthermore actually
    """.split()
)


def _topic_anchor_ok(original: str, candidate: str) -> bool:
    """Keep the first distinctive topic stem (libraries, electric, tourism, …)."""
    ordered = _ordered_content_stems(original)
    head = next(
        (s for s in ordered if len(s) >= 5 and s not in _WEAK_TOPIC_HEADS),
        None,
    )
    if not head:
        return True
    c_toks = _content_tokens(candidate)
    return head in c_toks


def _entity_injection_ok(original: str, candidate: str) -> bool:
    """Reject pronoun→new-noun inventions (fund them → fund their schools)."""
    o = original or ""
    c = candidate or ""
    o_toks = _content_tokens(o)
    for m in re.finditer(r"\b([A-Za-z]+)\s+(them|it|these|those)\b", o, flags=re.I):
        verb = m.group(1)
        m2 = re.search(
            rf"\b{re.escape(verb)}\s+(?:their|the|a|an|our|its)\s+([A-Za-z]+)\b",
            c,
            flags=re.I,
        )
        if not m2:
            continue
        noun = m2.group(1).lower()
        stem = noun
        if noun.endswith("ies") and len(noun) > 4:
            stem = noun[:-3] + "y"
        elif noun.endswith("es") and len(noun) > 4:
            stem = noun[:-1]
        elif noun.endswith("s") and len(noun) > 3:
            stem = noun[:-1]
        if stem not in o_toks and noun not in o_toks:
            return False
    return True


def _drift_ok(original: str, candidate: str) -> bool:
    """Reject rewrites that mostly introduce new content stems."""
    o_toks = _content_tokens(original)
    c_toks = _content_tokens(candidate)
    if len(c_toks) < 3:
        return True
    novel = c_toks - o_toks
    if len(novel) / len(c_toks) > 0.42:
        return False
    return True


_KNOWN_CONTRACTIONS = frozenset(
    {
        "don't",
        "doesn't",
        "didn't",
        "can't",
        "couldn't",
        "won't",
        "wouldn't",
        "shouldn't",
        "mustn't",
        "isn't",
        "aren't",
        "wasn't",
        "weren't",
        "hasn't",
        "haven't",
        "hadn't",
        "i'm",
        "i've",
        "i'd",
        "i'll",
        "you're",
        "you've",
        "you'd",
        "you'll",
        "we're",
        "we've",
        "we'd",
        "we'll",
        "they're",
        "they've",
        "they'd",
        "they'll",
        "it's",
        "that's",
        "what's",
        "who's",
        "where's",
        "there's",
        "here's",
        "let's",
    }
)


def _garbage_tokens(text: str) -> list[str]:
    """Flag glued contractions / nonsense tokens (e.g. don'tstet)."""
    bad: list[str] = []
    for raw in re.findall(r"[A-Za-z']+", text or ""):
        tok = raw.lower().strip("'")
        if not tok:
            continue
        # Contraction stem glued to extra letters: dontstet / don'tstet
        if re.match(
            r"(?:don|doesn|didn|won|can|isn|aren|wasn|weren|wouldn|shouldn|"
            r"mustn|hasn|haven|hadn)t[a-z]{2,}$",
            tok.replace("'", ""),
        ):
            bad.append(raw)
            continue
        if "'" in raw.lower() and raw.lower() not in _KNOWN_CONTRACTIONS:
            # Unknown apostrophe form longer than normal contractions
            if len(tok) >= 8:
                bad.append(raw)
                continue
        if re.search(r"(.)\1{3,}", tok):
            bad.append(raw)
    return bad


def _broken_rewrite_ok(original: str, candidate: str) -> bool:
    """Reject common FLAN tautologies / repeated n-grams / garbage tokens."""
    from collections import Counter

    c_low = (candidate or "").lower()
    o_low = (original or "").lower()
    for pat in (
        r"\bthan it would\b",
        r"\bthan they would\b",
        r"\bthan he would\b",
        r"\bthan she would\b",
    ):
        if re.search(pat, c_low) and not re.search(pat, o_low):
            return False
    if _garbage_tokens(candidate):
        return False
    # Hollow FLAN: "bills or bills", "ONS bills"
    if re.search(r"\b([a-z']{3,})\s+or\s+\1\b", c_low):
        return False
    o_words = {w.lower() for w in re.findall(r"[A-Za-z']+", original or "")}
    _ok_caps = {
        "USA",
        "UK",
        "EU",
        "UN",
        "AI",
        "ID",
        "PC",
        "TV",
        "DNA",
        "CPU",
        "GPU",
        "API",
        "PDF",
        "HTML",
        "HTTP",
        "HTTPS",
        "SQL",
    }
    for raw in re.findall(r"[A-Za-z']+", candidate or ""):
        if (
            raw.isupper()
            and 2 <= len(raw) <= 4
            and raw not in _ok_caps
            and raw.lower() not in o_words
        ):
            return False
    words = re.findall(r"[a-z']+", c_low)
    if len(words) >= 6:
        grams = [" ".join(words[i : i + 3]) for i in range(len(words) - 2)]
        for g, n in Counter(grams).items():
            if n >= 2 and len(g) >= 10:
                return False
    return True


def _grammar_not_worse(original: str, candidate: str) -> bool:
    """Reject if local grammar-issue count rises sharply vs the source unit."""
    try:
        from app.pipeline.grammar import check_rules
    except Exception:
        return True
    try:
        o_n = len(check_rules(original or ""))
        c_n = len(check_rules(candidate or ""))
    except Exception:
        return True
    # Allow one extra nit; block clear regressions (SV errors, glued tokens, …)
    return c_n <= o_n + 1


def _invention_ok(original: str, candidate: str) -> bool:
    """Reject candidates that invent content / topics not present in the source."""
    # Sentiment pads + common Small-LM topic inventions
    _PAD = frozenset(
        {
            "unfulfill",
            "unfulfilled",
            "miserable",
            "wonderful",
            "amazing",
            "fantastic",
            "terrible",
            "horrible",
            "beautiful",
            "awful",
            "delightful",
            "tragic",
            "glorious",
            "joyous",
            "blissful",
            "depress",
            "exciting",
            "thrilling",
            "fulfilling",
            "unsatisfying",
            "satisfying",
            "education",
            "awareness",
            "routine",
            "routines",
            "curriculum",
            "training",
            "workshop",
            "knowledge",
            "additionally",
            "furthermore",
            "moreover",
            # fitness / wellness pads FLAN invents on sedentary/TV sentences
            "workout",
            "workouts",
            "gym",
            "fitness",
            "regimen",
            "regime",
            "wellbeing",
            "wellness",
            "lifestyle",
            "nutrition",
            "diet",
            "diets",
            "calorie",
            "calories",
            "yoga",
            "cardio",
        }
    )
    o_toks = _content_tokens(original)
    c_toks = _content_tokens(candidate)
    if not c_toks:
        return True
    novel = c_toks - o_toks
    # Block known evaluative / topic pads not grounded in the source
    if novel & _PAD:
        return False
    # Block invented "… program/plan" fitness phrasing (e.g. "workout program")
    if "program" in novel or "programme" in novel:
        if novel & {"exercise", "fitness", "gym", "workout", "training"}:
            return False
    # Invented "learn/learning to <verb>" when verb not in source (e.g. learning to read)
    for m in re.finditer(
        r"\b(?:learn(?:ing)?|study(?:ing)?)\s+to\s+([a-zA-Z]+)\b",
        (candidate or ""),
        flags=re.I,
    ):
        verb = m.group(1).lower()
        stem = verb
        if stem.endswith("ing") and len(stem) > 5:
            stem = stem[:-3]
        elif stem.endswith("ed") and len(stem) > 4:
            stem = stem[:-2]
        elif stem.endswith("s") and len(stem) > 3:
            stem = stem[:-1]
        if stem not in o_toks and verb not in o_toks:
            return False
    # Invented "check with …" meeting frame not in source
    if re.search(r"\bcheck with\b", candidate or "", flags=re.I) and not re.search(
        r"\bcheck with\b", original or "", flags=re.I
    ):
        return False
    return True


def _meaning_score(original: str, candidate: str) -> float | None:
    """MiniLM cosine if available; None if model missing."""
    try:
        from app.pipeline.minilm import _cosine, _embed_texts, _ensure_model

        if _ensure_model() is None:
            return None
        vecs = _embed_texts([original, candidate])
        if len(vecs) < 2:
            return None
        return float(_cosine(vecs[0], vecs[1]))
    except Exception:
        return None


def validate_candidate(
    original: str,
    candidate: str,
    *,
    min_meaning: float = 0.80,
    max_surface: float = 0.92,
    min_surface: float = 0.25,
) -> ValidationResult:
    """Return whether a candidate is safe to keep as a rewrite of `original`."""
    reasons: list[str] = []
    o = (original or "").strip()
    c = (candidate or "").strip()
    if not o or not c:
        return ValidationResult(False, ["empty"], 0.0, 0.0)

    if c == o:
        return ValidationResult(False, ["identical"], 1.0, 1.0)

    if not polarity_safe(o, c):
        reasons.append("polarity")

    if not _length_ok(o, c):
        reasons.append("length")

    if not _unit_shape_ok(o, c):
        reasons.append("shape")

    if not _coverage_ok(o, c):
        reasons.append("coverage")

    if not _key_content_ok(o, c):
        reasons.append("key_content")

    if not _topic_anchor_ok(o, c):
        reasons.append("topic_anchor")

    if not _entity_injection_ok(o, c):
        reasons.append("entity_inject")

    if not _drift_ok(o, c):
        reasons.append("drift")

    if not _broken_rewrite_ok(o, c):
        reasons.append("broken")

    if not _grammar_not_worse(o, c):
        reasons.append("grammar_worse")

    if not _invention_ok(o, c):
        reasons.append("invention")

    surf = _surface_sim(o, c)
    if surf >= max_surface:
        reasons.append("too_similar")
    if surf < min_surface and len(o.split()) >= 8:
        reasons.append("too_divergent")

    o_nums, c_nums = _extract_numbers(o), _extract_numbers(c)
    if o_nums and not o_nums.issubset(c_nums):
        # Allow reformatting like 1,000 vs 1000 by normalizing commas
        o_norm = {n.replace(",", "") for n in o_nums}
        c_norm = {n.replace(",", "") for n in c_nums}
        if not o_norm.issubset(c_norm):
            reasons.append("numbers")

    o_quotes = _extract_quotes(o)
    c_quotes = _extract_quotes(c)
    if o_quotes and not o_quotes.issubset(c_quotes):
        reasons.append("quotes")

    for name in _extract_propers(o):
        if name not in c and name.lower() not in c.lower():
            reasons.append(f"entity:{name}")
            break

    meaning = _meaning_score(o, c)
    if meaning is None:
        # Without MiniLM, do not auto-pass when other gates already failed.
        meaning = 1.0 if not reasons else 0.5
    elif meaning < min_meaning:
        reasons.append(f"meaning:{meaning:.2f}")

    return ValidationResult(
        ok=not reasons,
        reasons=reasons,
        meaning=float(meaning),
        surface_sim=surf,
    )


def filter_valid_candidates(
    original: str,
    candidates: list[str],
    *,
    min_meaning: float = 0.80,
    max_surface: float = 0.92,
) -> list[tuple[str, ValidationResult]]:
    """Deduplicate and keep only candidates that pass validation."""
    seen: set[str] = set()
    out: list[tuple[str, ValidationResult]] = []
    for raw in candidates:
        text = re.sub(r"\s+", " ", (raw or "").strip())
        if not text:
            continue
        key = text.lower()
        if key in seen:
            continue
        seen.add(key)
        result = validate_candidate(
            original, text, min_meaning=min_meaning, max_surface=max_surface
        )
        if result.ok:
            out.append((text, result))
    return out