| """Display-only Hebrew text helpers for the Space. No network, no rag imports.""" |
|
|
| from __future__ import annotations |
|
|
| import re |
| from dataclasses import dataclass |
|
|
| _TOKEN_RE = re.compile(r"[0-9A-Za-z\u0590-\u05FF]+") |
| _WORD = "0-9A-Za-z\u0590-\u05FF" |
| _SENT_SPLIT = re.compile(r"(?<=[\.!\?\u2026])\s+|\n+") |
|
|
|
|
| @dataclass(frozen=True) |
| class DisplayOpts: |
| max_quote_chars: int |
| ellipsis: str |
| min_term_chars: int |
| min_stem_chars: int |
| max_prefix_chars: int |
| stopwords: frozenset[str] |
| prefixes: tuple[str, ...] |
|
|
|
|
| def strip_fffd(text: str) -> str: |
| return (text or "").replace("\ufffd", "") |
|
|
|
|
| def split_sentences(text: str) -> list[str]: |
| cleaned = strip_fffd(text).replace("\r\n", "\n").replace("\r", "\n") |
| parts = _SENT_SPLIT.split(cleaned) |
| out: list[str] = [] |
| for part in parts: |
| norm = re.sub(r"\s+", " ", part).strip() |
| if norm: |
| out.append(norm) |
| return out |
|
|
|
|
| def _sentence_key(sentence: str) -> str: |
| collapsed = re.sub(r"\s+", " ", sentence).strip() |
| return collapsed.strip(" \t.!?;:\u2026").casefold() |
|
|
|
|
| def collapse_immediate_repeats(sentences: list[str]) -> list[str]: |
| out: list[str] = [] |
| prev = None |
| for sentence in sentences: |
| key = _sentence_key(sentence) |
| if prev is not None and key == prev: |
| continue |
| out.append(sentence) |
| prev = key |
| return out |
|
|
|
|
| def server_spans_survive(text: str, max_quote_chars: int | None) -> bool: |
| """True when display will not rewrite the quote, so code-point spans stay valid. |
| |
| U+FFFD, an immediate repeated sentence, or a trim to max_quote_chars changes |
| the string and shifts offsets. A quote that already fits is left as-is. |
| """ |
| if "\ufffd" in text: |
| return False |
| sentences = split_sentences(text) |
| if len(collapse_immediate_repeats(sentences)) < len(sentences): |
| return False |
| if max_quote_chars is not None and len(text) > max_quote_chars: |
| return False |
| return True |
|
|
|
|
| def _allowed_prefixes(opts: DisplayOpts) -> list[str]: |
| seen: set[str] = set() |
| ordered: list[str] = [] |
| for prefix in sorted(opts.prefixes, key=len, reverse=True): |
| if not prefix or len(prefix) > opts.max_prefix_chars or prefix in seen: |
| continue |
| seen.add(prefix) |
| ordered.append(prefix) |
| return ordered |
|
|
|
|
| def strip_prefix(token: str, opts: DisplayOpts) -> str: |
| """Longest configured clitic whose remainder is still a real stem.""" |
| folded = (token or "").casefold() |
| for prefix in _allowed_prefixes(opts): |
| if not folded.startswith(prefix): |
| continue |
| stem = folded[len(prefix) :] |
| if len(stem) >= opts.min_stem_chars: |
| return stem |
| return "" |
|
|
|
|
| def content_terms(question: str, opts: DisplayOpts) -> list[str]: |
| out: list[str] = [] |
| seen: set[str] = set() |
|
|
| def add(token: str) -> None: |
| folded = token.casefold() |
| if len(folded) < opts.min_term_chars or folded in opts.stopwords or folded in seen: |
| return |
| seen.add(folded) |
| out.append(folded) |
|
|
| for token in _TOKEN_RE.findall(question or ""): |
| add(token) |
| stem = strip_prefix(token, opts) |
| if stem: |
| add(stem) |
| return out |
|
|
|
|
| def _term_pattern(term: str, opts: DisplayOpts) -> re.Pattern[str]: |
| prefixes = _allowed_prefixes(opts) |
| if prefixes: |
| alt = "|".join(re.escape(prefix) for prefix in prefixes) |
| optional = f"(?:{alt})?" |
| else: |
| optional = "" |
| return re.compile( |
| rf"(?<![{_WORD}]){optional}{re.escape(term)}(?![{_WORD}])", |
| re.IGNORECASE, |
| ) |
|
|
|
|
| def sentence_matches(sentence: str, terms: list[str], opts: DisplayOpts) -> bool: |
| if not sentence or not terms: |
| return False |
| for term in terms: |
| if _term_pattern(term, opts).search(sentence): |
| return True |
| return False |
|
|
|
|
| def trim_at_word_boundary(text: str, max_chars: int, ellipsis: str) -> str: |
| """Cut to about max_chars on a whitespace boundary and append ellipsis.""" |
| collapsed = re.sub(r"\s+", " ", text or "").strip() |
| if max_chars <= 0: |
| return ellipsis if collapsed else "" |
| if len(collapsed) <= max_chars: |
| return collapsed |
| room = max_chars - len(ellipsis) |
| if room <= 0: |
| return ellipsis |
| window = collapsed[:room] |
| cut = window.rfind(" ") |
| if cut > 0: |
| window = window[:cut] |
| return window.rstrip() + ellipsis |
|
|
|
|
| def prepare_quote(quote: str, question: str, opts: DisplayOpts) -> str: |
| """Sentences that hit the question, repeats collapsed, then word-trimmed.""" |
| sentences = collapse_immediate_repeats(split_sentences(quote)) |
| terms = content_terms(question, opts) |
| if terms: |
| matched = [sentence for sentence in sentences if sentence_matches(sentence, terms, opts)] |
| |
| if matched: |
| sentences = matched |
| return trim_at_word_boundary(" ".join(sentences), opts.max_quote_chars, opts.ellipsis) |
|
|
|
|
| def highlight_terms(text: str, question: str, opts: DisplayOpts) -> str: |
| """Escape text and wrap matched terms, including configured Hebrew prefixes.""" |
| import html |
|
|
| plain = text or "" |
| terms = sorted(content_terms(question, opts), key=len, reverse=True) |
| spans: list[tuple[int, int]] = [] |
| for term in terms: |
| for match in _term_pattern(term, opts).finditer(plain): |
| spans.append((match.start(), match.end())) |
| spans.sort(key=lambda span: (span[0], -(span[1] - span[0]))) |
| chosen: list[tuple[int, int]] = [] |
| cursor = 0 |
| for start, end in spans: |
| if start < cursor: |
| continue |
| chosen.append((start, end)) |
| cursor = end |
| pieces: list[str] = [] |
| index = 0 |
| for start, end in chosen: |
| pieces.append(html.escape(plain[index:start], quote=True)) |
| pieces.append("<strong>" + html.escape(plain[start:end], quote=True) + "</strong>") |
| index = end |
| pieces.append(html.escape(plain[index:], quote=True)) |
| return "".join(pieces) |
|
|
