Blame for textproc.py

"""Display-only Hebrew text helpers for the Space. No network, no rag imports."""

from __future__ import annotations

import re
from dataclasses import dataclass

_TOKEN_RE = re.compile(r"[0-9A-Za-z\u0590-\u05FF]+")
_WORD = "0-9A-Za-z\u0590-\u05FF"
_SENT_SPLIT = re.compile(r"(?<=[\.!\?\u2026])\s+|\n+")


@dataclass(frozen=True)
class DisplayOpts:
    max_quote_chars: int
    ellipsis: str
    min_term_chars: int
    min_stem_chars: int
    max_prefix_chars: int
    stopwords: frozenset[str]
    prefixes: tuple[str, ...]


def strip_fffd(text: str) -> str:
    return (text or "").replace("\ufffd", "")


def split_sentences(text: str) -> list[str]:
    cleaned = strip_fffd(text).replace("\r\n", "\n").replace("\r", "\n")
    parts = _SENT_SPLIT.split(cleaned)
    out: list[str] = []
    for part in parts:
        norm = re.sub(r"\s+", " ", part).strip()
        if norm:
            out.append(norm)
    return out


def _sentence_key(sentence: str) -> str:
    collapsed = re.sub(r"\s+", " ", sentence).strip()
    return collapsed.strip(" \t.!?;:\u2026").casefold()


def collapse_immediate_repeats(sentences: list[str]) -> list[str]:
    out: list[str] = []
    prev = None
    for sentence in sentences:
        key = _sentence_key(sentence)
        if prev is not None and key == prev:
            continue
        out.append(sentence)
        prev = key
    return out


def server_spans_survive(text: str, max_quote_chars: int | None) -> bool:
    """True when display will not rewrite the quote, so code-point spans stay valid.

    U+FFFD, an immediate repeated sentence, or a trim to max_quote_chars changes
    the string and shifts offsets. A quote that already fits is left as-is.
    """
    if "\ufffd" in text:
        return False
    sentences = split_sentences(text)
    if len(collapse_immediate_repeats(sentences)) < len(sentences):
        return False
    if max_quote_chars is not None and len(text) > max_quote_chars:
        return False
    return True


def _allowed_prefixes(opts: DisplayOpts) -> list[str]:
    seen: set[str] = set()
    ordered: list[str] = []
    for prefix in sorted(opts.prefixes, key=len, reverse=True):
        if not prefix or len(prefix) > opts.max_prefix_chars or prefix in seen:
            continue
        seen.add(prefix)
        ordered.append(prefix)
    return ordered


def strip_prefix(token: str, opts: DisplayOpts) -> str:
    """Longest configured clitic whose remainder is still a real stem."""
    folded = (token or "").casefold()
    for prefix in _allowed_prefixes(opts):
        if not folded.startswith(prefix):
            continue
        stem = folded[len(prefix) :]
        if len(stem) >= opts.min_stem_chars:
            return stem
    return ""


def content_terms(question: str, opts: DisplayOpts) -> list[str]:
    out: list[str] = []
    seen: set[str] = set()

    def add(token: str) -> None:
        folded = token.casefold()
        if len(folded) < opts.min_term_chars or folded in opts.stopwords or folded in seen:
            return
        seen.add(folded)
        out.append(folded)

    for token in _TOKEN_RE.findall(question or ""):
        add(token)
        stem = strip_prefix(token, opts)
        if stem:
            add(stem)
    return out


def _term_pattern(term: str, opts: DisplayOpts) -> re.Pattern[str]:
    prefixes = _allowed_prefixes(opts)
    if prefixes:
        alt = "|".join(re.escape(prefix) for prefix in prefixes)
        optional = f"(?:{alt})?"
    else:
        optional = ""
    return re.compile(
        rf"(?<![{_WORD}]){optional}{re.escape(term)}(?![{_WORD}])",
        re.IGNORECASE,
    )


def sentence_matches(sentence: str, terms: list[str], opts: DisplayOpts) -> bool:
    if not sentence or not terms:
        return False
    for term in terms:
        if _term_pattern(term, opts).search(sentence):
            return True
    return False


def trim_at_word_boundary(text: str, max_chars: int, ellipsis: str) -> str:
    """Cut to about max_chars on a whitespace boundary and append ellipsis."""
    collapsed = re.sub(r"\s+", " ", text or "").strip()
    if max_chars <= 0:
        return ellipsis if collapsed else ""
    if len(collapsed) <= max_chars:
        return collapsed
    room = max_chars - len(ellipsis)
    if room <= 0:
        return ellipsis
    window = collapsed[:room]
    cut = window.rfind(" ")
    if cut > 0:
        window = window[:cut]
    return window.rstrip() + ellipsis


def prepare_quote(quote: str, question: str, opts: DisplayOpts) -> str:
    """Sentences that hit the question, repeats collapsed, then word-trimmed."""
    sentences = collapse_immediate_repeats(split_sentences(quote))
    terms = content_terms(question, opts)
    if terms:
        matched = [sentence for sentence in sentences if sentence_matches(sentence, terms, opts)]
        # No sentence hit the question: keep the collapsed quote so the card is not blank.
        if matched:
            sentences = matched
    return trim_at_word_boundary(" ".join(sentences), opts.max_quote_chars, opts.ellipsis)


def highlight_terms(text: str, question: str, opts: DisplayOpts) -> str:
    """Escape text and wrap matched terms, including configured Hebrew prefixes."""
    import html

    plain = text or ""
    terms = sorted(content_terms(question, opts), key=len, reverse=True)
    spans: list[tuple[int, int]] = []
    for term in terms:
        for match in _term_pattern(term, opts).finditer(plain):
            spans.append((match.start(), match.end()))
    spans.sort(key=lambda span: (span[0], -(span[1] - span[0])))
    chosen: list[tuple[int, int]] = []
    cursor = 0
    for start, end in spans:
        if start < cursor:
            continue
        chosen.append((start, end))
        cursor = end
    pieces: list[str] = []
    index = 0
    for start, end in chosen:
        pieces.append(html.escape(plain[index:start], quote=True))
        pieces.append("<strong>" + html.escape(plain[start:end], quote=True) + "</strong>")
        index = end
    pieces.append(html.escape(plain[index:], quote=True))
    return "".join(pieces)