"""Display-only Hebrew text helpers for the Space. No network, no rag imports.""" from __future__ import annotations import re from dataclasses import dataclass _TOKEN_RE = re.compile(r"[0-9A-Za-z\u0590-\u05FF]+") _WORD = "0-9A-Za-z\u0590-\u05FF" _SENT_SPLIT = re.compile(r"(?<=[\.!\?\u2026])\s+|\n+") @dataclass(frozen=True) class DisplayOpts: max_quote_chars: int ellipsis: str min_term_chars: int min_stem_chars: int max_prefix_chars: int stopwords: frozenset[str] prefixes: tuple[str, ...] def strip_fffd(text: str) -> str: return (text or "").replace("\ufffd", "") def split_sentences(text: str) -> list[str]: cleaned = strip_fffd(text).replace("\r\n", "\n").replace("\r", "\n") parts = _SENT_SPLIT.split(cleaned) out: list[str] = [] for part in parts: norm = re.sub(r"\s+", " ", part).strip() if norm: out.append(norm) return out def _sentence_key(sentence: str) -> str: collapsed = re.sub(r"\s+", " ", sentence).strip() return collapsed.strip(" \t.!?;:\u2026").casefold() def collapse_immediate_repeats(sentences: list[str]) -> list[str]: out: list[str] = [] prev = None for sentence in sentences: key = _sentence_key(sentence) if prev is not None and key == prev: continue out.append(sentence) prev = key return out def server_spans_survive(text: str, max_quote_chars: int | None) -> bool: """True when display will not rewrite the quote, so code-point spans stay valid. U+FFFD, an immediate repeated sentence, or a trim to max_quote_chars changes the string and shifts offsets. A quote that already fits is left as-is. """ if "\ufffd" in text: return False sentences = split_sentences(text) if len(collapse_immediate_repeats(sentences)) < len(sentences): return False if max_quote_chars is not None and len(text) > max_quote_chars: return False return True def _allowed_prefixes(opts: DisplayOpts) -> list[str]: seen: set[str] = set() ordered: list[str] = [] for prefix in sorted(opts.prefixes, key=len, reverse=True): if not prefix or len(prefix) > opts.max_prefix_chars or prefix in seen: continue seen.add(prefix) ordered.append(prefix) return ordered def strip_prefix(token: str, opts: DisplayOpts) -> str: """Longest configured clitic whose remainder is still a real stem.""" folded = (token or "").casefold() for prefix in _allowed_prefixes(opts): if not folded.startswith(prefix): continue stem = folded[len(prefix) :] if len(stem) >= opts.min_stem_chars: return stem return "" def content_terms(question: str, opts: DisplayOpts) -> list[str]: out: list[str] = [] seen: set[str] = set() def add(token: str) -> None: folded = token.casefold() if len(folded) < opts.min_term_chars or folded in opts.stopwords or folded in seen: return seen.add(folded) out.append(folded) for token in _TOKEN_RE.findall(question or ""): add(token) stem = strip_prefix(token, opts) if stem: add(stem) return out def _term_pattern(term: str, opts: DisplayOpts) -> re.Pattern[str]: prefixes = _allowed_prefixes(opts) if prefixes: alt = "|".join(re.escape(prefix) for prefix in prefixes) optional = f"(?:{alt})?" else: optional = "" return re.compile( rf"(? bool: if not sentence or not terms: return False for term in terms: if _term_pattern(term, opts).search(sentence): return True return False def trim_at_word_boundary(text: str, max_chars: int, ellipsis: str) -> str: """Cut to about max_chars on a whitespace boundary and append ellipsis.""" collapsed = re.sub(r"\s+", " ", text or "").strip() if max_chars <= 0: return ellipsis if collapsed else "" if len(collapsed) <= max_chars: return collapsed room = max_chars - len(ellipsis) if room <= 0: return ellipsis window = collapsed[:room] cut = window.rfind(" ") if cut > 0: window = window[:cut] return window.rstrip() + ellipsis def prepare_quote(quote: str, question: str, opts: DisplayOpts) -> str: """Sentences that hit the question, repeats collapsed, then word-trimmed.""" sentences = collapse_immediate_repeats(split_sentences(quote)) terms = content_terms(question, opts) if terms: matched = [sentence for sentence in sentences if sentence_matches(sentence, terms, opts)] # No sentence hit the question: keep the collapsed quote so the card is not blank. if matched: sentences = matched return trim_at_word_boundary(" ".join(sentences), opts.max_quote_chars, opts.ellipsis) def highlight_terms(text: str, question: str, opts: DisplayOpts) -> str: """Escape text and wrap matched terms, including configured Hebrew prefixes.""" import html plain = text or "" terms = sorted(content_terms(question, opts), key=len, reverse=True) spans: list[tuple[int, int]] = [] for term in terms: for match in _term_pattern(term, opts).finditer(plain): spans.append((match.start(), match.end())) spans.sort(key=lambda span: (span[0], -(span[1] - span[0]))) chosen: list[tuple[int, int]] = [] cursor = 0 for start, end in spans: if start < cursor: continue chosen.append((start, end)) cursor = end pieces: list[str] = [] index = 0 for start, end in chosen: pieces.append(html.escape(plain[index:start], quote=True)) pieces.append("" + html.escape(plain[start:end], quote=True) + "") index = end pieces.append(html.escape(plain[index:], quote=True)) return "".join(pieces)