fixers.py

"""Rule-based sentence boundary fixers for la_senter."""

from typing import Callable

from spacy.language import Language
from spacy.tokens import Doc, Token

CLOSING_QUOTES = {
    '"',          # straight double (U+0022) — ambiguous
    "”",     # right double quotation mark " — definitively closing
    "'",          # straight single (U+0027) — ambiguous
    "’",     # right single quotation mark ' — definitively closing
    "»",     # right-pointing double angle quotation mark » — definitively closing
    "›",     # single right-pointing angle quotation mark › — definitively closing
}

# These characters unambiguously encode direction — no whitespace check needed
DEFINITIVELY_CLOSING = {"”", "’", "»", "›"}

# Hard terminals after which a quote is a closer, not an opener.
# Colon is excluded: in Latin, colon introduces direct speech ("loquitur: 'Rex...'")
# and a quote after colon is always an opener.
CLOSING_TERMINALS = {".", "!", "?", ";"}


def _is_likely_closing(token: Token) -> bool:
    """Return True if a sentence-initial quote is closing an open speech.

    For unambiguous closing characters (curly quotes, guillemets): always True.
    For ambiguous straight/single quotes: True only when the immediately preceding
    token is a hard sentence terminal (. ! ? ;). Colon is excluded because in
    Latin it introduces direct speech and a quote after colon is an opener.
    """
    if token.text in DEFINITIVELY_CLOSING:
        return True
    return token.i > 0 and token.nbor(-1).text in CLOSING_TERMINALS


OPEN_BRACKETS = {"(", "["}
CLOSE_BRACKETS = {")", "]"}

# Unicode dash/hyphen characters used as parenthetical delimiters.
# Not added to OPEN_BRACKETS because bare - is too common in Latin
# (enclitic separators, verse markup) to treat as a bracket opener.
# Instead, _dash_paren_fix uses a stricter paired-with-terminal check.
DASH_CHARS = {
    "-",           # U+002D HYPHEN-MINUS
    "—",           # U+2014 EM DASH
    "–",           # U+2013 EN DASH
    "‐",           # U+2010 HYPHEN
    "‑",           # U+2011 NON-BREAKING HYPHEN
}
DASH_TERMINALS = {"!", "?"}


def paren_fix(doc: Doc) -> Doc:
    """Suppress sentence boundaries that fall inside unclosed parentheses/brackets.

    The senter treats '!' and '?' as terminals even inside parentheticals like
    '(mirabile dictu!)'. This rule scans backward from each predicted boundary
    to the previous sentence start, counting open vs. close brackets — if any
    bracket is still open, the boundary is spurious and is suppressed.
    """
    for token in doc[1:]:
        if not token.is_sent_start:
            continue
        depth = 0
        for t in reversed(list(doc[:token.i])):
            if t.text in CLOSE_BRACKETS:
                depth += 1
            elif t.text in OPEN_BRACKETS:
                if depth > 0:
                    depth -= 1
                else:
                    token.is_sent_start = False
                    break
            if t.is_sent_start and t.i > 0:
                break
    return doc


def _dash_paren_fix(doc: Doc) -> Doc:
    """Suppress boundaries where a dash-parenthetical contains a terminal.

    Handles '- content! -' and '— content? —' asides. Only fires when:
    1. The sent_start token is itself a dash character (the closing dash), AND
    2. Scanning backward finds an opening dash of any DASH_CHARS type, AND
    3. A '!' or '?' appears between the two dashes.

    This avoids the false-positive problem of bare '-' in Latin enclitic
    separators (e.g. '-que') which must never suppress a sentence boundary.
    """
    for token in doc[1:]:
        if not token.is_sent_start or token.text not in DASH_CHARS:
            continue
        found_terminal = False
        for t in reversed(list(doc[:token.i])):
            if t.is_sent_start and t.i > 0:
                break
            if t.text in DASH_TERMINALS:
                found_terminal = True
            elif t.text in DASH_CHARS and found_terminal:
                token.is_sent_start = False
                break
    return doc


def quote_fix(doc: Doc) -> Doc:
    for token in doc[1:]:
        if (
            token.is_sent_start
            and token.text in CLOSING_QUOTES
            and _is_likely_closing(token)
        ):
            token.is_sent_start = False
            if token.i + 1 < len(doc):
                doc[token.i + 1].is_sent_start = True
    return doc


def token_fix(doc: Doc) -> Doc:
    """Unified rules-based sentence boundary corrector for Latin text.

    Applies all punctuation-aware corrections in a single pipeline pass:
    - paren_fix: suppress boundaries inside unclosed parentheses/brackets
    - quote_fix: move boundaries past orphaned closing quotes

    Additional rules (em-dash parentheticals, etc.) should be added here.
    """
    doc = paren_fix(doc)
    doc = _dash_paren_fix(doc)
    doc = quote_fix(doc)
    return doc


@Language.factory("token_fix")
def create_token_fix(nlp: Language, name: str) -> Callable[[Doc], Doc]:
    """Factory for the token_fix pipeline component.

    Registered as a factory (not Language.component) so spaCy records this
    module as the factory's source. `spacy package` reads that module to derive
    model requirements; the component form records 'spacy.language' instead,
    which drops latincy-senter from the published model's dependencies.
    token_fix itself stays a plain Doc -> Doc callable.
    """
    return token_fix