app/engine/orchestrator.py

"""Batch/stream orchestrator for the structural rewrite engine."""

from __future__ import annotations

import logging
import time
from difflib import SequenceMatcher

from app.config import (
    ENGINE_BATCH_PARAS,
    ENGINE_CLASSICAL_AGGRESSIVE,
    ENGINE_FORCE_REWRITE,
    ENGINE_LEXICAL_MIN_WSD,
    ENGINE_LEXICAL_REFINEMENT,
    ENGINE_MIN_CONFIDENCE,
    ENGINE_PARAPHRASE,
    ENGINE_PARAPHRASE_MIN_SIM,
    ENGINE_PARAPHRASE_PRIMARY,
    ENGINE_PHRASE_MAX_CHANGES,
    ENGINE_PHRASE_MIN_SIM,
    ENGINE_PHRASE_REWRITE,
    ENGINE_PHRASE_USE_T5,
    ENGINE_PRESERVE_LENGTH,
    ENGINE_REQUIRE_WORDING_CHANGE,
    ENGINE_SAFETY_MIN,
    ENGINE_SPLIT_LONG,
    ENGINE_SPLIT_MIN_WORDS,
    ENGINE_STRUCTURAL_VARIATION,
    ENGINE_USE_MINILM_SAFETY,
    GRAMMAR_FIX_OUTPUT,
)
from app.engine.classify import classify_sentence
from app.engine.consistency import apply_consistency
from app.engine.force import force_cleft_rewrite
from app.engine.grammar import repair_sentence
from app.engine.ingest import ingest_text
from app.engine.lexical import (
    dynamic_lexical_budget,
    ensure_wording_change,
    refine_sentence,
)
from app.engine.mechanics import enforce_length_budget, tidy
from app.engine.phrase import rewrite_phrases
from app.engine.models import (
    DocumentBlock,
    EngineResult,
    EngineStats,
    SentenceRecord,
)
from app.engine.normalize import (
    detect_blocks,
    mask_protected_spans,
    normalize_text,
    unmask_protected_spans,
)
from app.engine.fallback import structural_fallback_candidates
from app.engine.paraphrase import (
    paraphrase_sentence,
    sufficiently_changed,
    surface_similarity,
)
from app.engine.plan import build_plan
from app.engine.rewrite import generate_from_plan
from app.engine.safety import check_safety
from app.engine.segment import iter_paragraph_batches, split_sentences, word_count
from app.engine.stitch import join_sentences, stitch_blocks
from app.engine.variation import new_seed, rotate
from app.engine.voice import active_to_passive

logger = logging.getLogger("plainrewrite.engine")

_REWRITEABLE_TYPES = frozenset(
    {"simple_declarative", "compound", "complex", "because_clause"}
)


def _classical_modes(use_minilm: bool) -> tuple[bool, bool]:
    """Return (classical_strict, classical_aggressive) for wording paths."""
    if use_minilm:
        return False, False
    return True, bool(ENGINE_CLASSICAL_AGGRESSIVE)


def _similarity(a: str, b: str) -> float:
    return SequenceMatcher(None, (a or "").lower(), (b or "").lower()).ratio()


def _apply_lexical_refinement(

    record: SentenceRecord,

    *,

    enabled: bool,

    min_wsd: float,

    max_changes: int | None,

    safety_min: float,

    min_confidence: float,

    use_minilm: bool,

    polish: bool = False,

) -> SentenceRecord:
    if not enabled or record.sentence_type not in _REWRITEABLE_TYPES:
        return record
    before = record.rewritten
    classical_strict, classical_aggressive = _classical_modes(use_minilm)
    refined = refine_sentence(
        before,
        min_wsd=min_wsd,
        max_changes=max_changes,
        polish=polish,
        classical_strict=classical_strict,
        classical_aggressive=classical_aggressive,
    )
    if not refined.changes or refined.text == before:
        return record
    safety = check_safety(
        before,
        refined.text,
        min_meaning=safety_min,
        min_confidence=min_confidence,
        use_minilm=use_minilm,
        protected_entities=None,
        protected_auxiliaries=None,
        structural_validation=False,
    )
    if not safety.ok:
        return record

    prior_status = record.status
    prior_reasons = list(record.reasons)
    prior_lex = list(record.lexical_changes)
    record.rewritten = refined.text
    record.lexical_changes = prior_lex + refined.changes
    record.status = "rewritten"
    if not record.template_id:
        record.template_id = "lexical_refine"
    elif "+lexical" not in record.template_id:
        record.template_id = f"{record.template_id}+lexical"
    if prior_status in {"skipped", "reverted"}:
        record.reasons = [
            f"structural_{prior_status}:{reason}" for reason in prior_reasons
        ]
    record.confidence = min(
        safety.confidence,
        record.confidence if record.confidence > 0 else max(min_confidence, 0.55),
    )
    return record


def _apply_extra_polish(

    record: SentenceRecord,

    *,

    enabled: bool,

    min_wsd: float,

    max_changes: int | None,

    safety_min: float,

    min_confidence: float,

    use_minilm: bool,

) -> SentenceRecord:
    """Polish-only extra pass so the UI toggle has a real end-to-end effect."""
    if not enabled or record.sentence_type not in _REWRITEABLE_TYPES:
        return record
    classical_strict, classical_aggressive = _classical_modes(use_minilm)
    # Tight classical (no MiniLM, not aggressive): skip extra polish cycling.
    if classical_strict and not classical_aggressive:
        return record
    before = record.rewritten
    target_budget = (
        max(1, min(int(max_changes), 15))
        if max_changes is not None
        else dynamic_lexical_budget(before, polish=True)
    )
    remaining = max(0, target_budget - len(record.lexical_changes))
    if remaining <= 0:
        return record
    refined = refine_sentence(
        before,
        min_wsd=min(min_wsd, 0.10),
        max_changes=remaining,
        aggressive=True,
        classical_strict=classical_strict,
        classical_aggressive=classical_aggressive,
    )
    if not refined.changes or refined.text == before:
        return record
    safety = check_safety(
        before,
        refined.text,
        min_meaning=safety_min,
        min_confidence=min_confidence,
        use_minilm=use_minilm,
        protected_entities=None,
        protected_auxiliaries=None,
        structural_validation=False,
    )
    if not safety.ok:
        return record

    prior_lex = list(record.lexical_changes)
    record.rewritten = refined.text
    record.lexical_changes = prior_lex + refined.changes
    record.status = "rewritten"
    if not record.template_id:
        record.template_id = "lexical_refine"
    elif "+lexical" not in record.template_id:
        record.template_id = f"{record.template_id}+lexical"
    record.confidence = min(
        safety.confidence,
        record.confidence if record.confidence > 0 else max(min_confidence, 0.55),
    )
    return record


def _apply_phrase_rewrite(

    record: SentenceRecord,

    *,

    enabled: bool,

    polish: bool,

    safety_min: float,

    min_confidence: float,

    use_minilm: bool,

) -> SentenceRecord:
    """Rewrite verb–object phrases before single-word lexical polish."""
    if not enabled or record.sentence_type not in _REWRITEABLE_TYPES:
        return record
    before = record.rewritten
    max_changes = (
        ENGINE_PHRASE_MAX_CHANGES
        if ENGINE_PHRASE_MAX_CHANGES > 0
        else (2 if polish else 1)
    )
    classical_strict, classical_aggressive = _classical_modes(use_minilm)
    refined = rewrite_phrases(
        before,
        max_changes=max_changes,
        polish=polish,
        min_sim=min(ENGINE_PHRASE_MIN_SIM, safety_min),
        use_t5=ENGINE_PHRASE_USE_T5 and use_minilm,
        classical_strict=classical_strict,
        classical_aggressive=classical_aggressive,
    )
    if not refined.changes or refined.text == before:
        return record
    safety = check_safety(
        before,
        refined.text,
        min_meaning=min(safety_min, ENGINE_PHRASE_MIN_SIM),
        min_confidence=min_confidence,
        use_minilm=use_minilm,
        protected_entities=None,
        protected_auxiliaries=None,
        structural_validation=False,
    )
    if not safety.ok:
        return record

    prior_lex = list(record.lexical_changes)
    record.rewritten = refined.text
    record.lexical_changes = prior_lex + refined.changes
    record.status = "rewritten"
    if not record.template_id:
        record.template_id = "phrase_rewrite"
    elif "+phrase" not in record.template_id:
        record.template_id = f"{record.template_id}+phrase"
    record.confidence = min(
        safety.confidence,
        record.confidence if record.confidence > 0 else max(min_confidence, 0.55),
    )
    return record


def _unchanged(record: SentenceRecord) -> bool:
    return (
        record.rewritten.strip().lower().rstrip(".!?")
        == record.original.strip().lower().rstrip(".!?")
    )


def _try_accept_rewrite(

    record: SentenceRecord,

    candidate: str,

    *,

    template_id: str,

    confidence: float,

    safety_min: float,

    min_confidence: float,

    use_minilm: bool,

    reason_prefix: str,

) -> SentenceRecord | None:
    """Accept a candidate when safety passes and wording truly changed."""
    if not sufficiently_changed(record.original, candidate):
        return None
    safety = check_safety(
        record.original,
        candidate,
        min_meaning=min(safety_min, ENGINE_PARAPHRASE_MIN_SIM),
        min_confidence=min_confidence,
        use_minilm=use_minilm,
        protected_entities=None,
        protected_auxiliaries=None,
        structural_validation=False,
    )
    if not safety.ok:
        hard = check_safety(
            record.original,
            candidate,
            hard_invariants_only=True,
        )
        if not hard.ok:
            return None
        safety = hard

    repaired = repair_sentence(candidate) if GRAMMAR_FIX_OUTPUT else candidate
    if repaired != candidate:
        repaired_safety = check_safety(
            record.original,
            repaired,
            hard_invariants_only=True,
        )
        if repaired_safety.ok and sufficiently_changed(
            record.original, repaired
        ):
            candidate = repaired
            safety = repaired_safety

    prior_status = record.status
    prior_reasons = list(record.reasons)
    record.rewritten = candidate
    record.status = "rewritten"
    record.template_id = template_id
    record.confidence = min(0.75, max(confidence, safety.confidence))
    record.reasons = [
        f"{reason_prefix}_from_{prior_status}:{reason}"
        for reason in prior_reasons
    ] or [reason_prefix]
    return record


def _apply_paraphrase_fallback(

    record: SentenceRecord,

    *,

    enabled: bool,

    safety_min: float,

    min_confidence: float,

    use_minilm: bool,

    primary: bool | None = None,

    variation_seed: int | None = None,

) -> SentenceRecord:
    """Meaning-safe paraphrase rewrite, then structural fallbacks if needed.



    When primary mode is on, paraphrase can replace an existing light structural

    rewrite if the paraphrase keeps meaning and looks substantially newer.

    """
    if record.sentence_type not in _REWRITEABLE_TYPES:
        return record
    prefer_primary = ENGINE_PARAPHRASE_PRIMARY if primary is None else primary

    options: list[tuple[str, str, float, str]] = []
    if enabled:
        result = paraphrase_sentence(
            record.original,
            prefer_divergent=prefer_primary,
        )
        if (
            result.text
            and result.text != record.original
            and sufficiently_changed(record.original, result.text)
        ):
            para_sim = surface_similarity(record.original, result.text)
            current_sim = surface_similarity(record.original, record.rewritten)
            # Always keep paraphrase when the sentence is still unchanged.
            # In primary mode, also prefer it when it is clearly more divergent
            # than the current structural rewrite.
            take_paraphrase = _unchanged(record) or (
                prefer_primary and para_sim + 0.03 < current_sim
            )
            if take_paraphrase:
                options.append(
                    (
                        "paraphrase",
                        result.text,
                        max(0.55, result.confidence),
                        "paraphrase_primary"
                        if prefer_primary
                        else "paraphrase_fallback",
                    )
                )

    # Structural fallbacks only fill gaps. If the plan path already rewrote the
    # sentence, overwriting it here would discard per-request rotation and pin
    # every sentence to the single highest-ranked fallback.
    if _unchanged(record):
        structural = rotate(
            structural_fallback_candidates(record.original),
            seed=variation_seed,
            position=record.index,
            confidence_of=lambda item: item[2],
        )
        for template_id, candidate, confidence in structural:
            options.append(
                (template_id, candidate, confidence, "structure_fallback")
            )

    for template_id, candidate, confidence, reason_prefix in options:
        accepted = _try_accept_rewrite(
            record,
            candidate,
            template_id=template_id,
            confidence=confidence,
            safety_min=safety_min,
            min_confidence=min_confidence,
            use_minilm=use_minilm,
            reason_prefix=reason_prefix,
        )
        if accepted is not None:
            return accepted
    return record


def _apply_forced_rewrite(record: SentenceRecord, *, enabled: bool) -> SentenceRecord:
    """Optional legacy cleft rewrite; disabled by default."""
    if not enabled or record.sentence_type not in _REWRITEABLE_TYPES:
        return record
    if not _unchanged(record):
        return record

    candidate = force_cleft_rewrite(record.original)
    if not candidate:
        return record
    safety = check_safety(
        record.original,
        candidate,
        hard_invariants_only=True,
    )
    if not safety.ok:
        return record
    repaired = repair_sentence(candidate) if GRAMMAR_FIX_OUTPUT else candidate
    if repaired != candidate:
        repaired_safety = check_safety(
            record.original,
            repaired,
            hard_invariants_only=True,
        )
        if not repaired_safety.ok:
            repaired = candidate
        else:
            safety = repaired_safety

    prior_status = record.status
    prior_reasons = list(record.reasons)
    record.rewritten = repaired
    record.status = "rewritten"
    record.template_id = "forced_cleft"
    record.confidence = min(0.60, safety.confidence)
    record.reasons = [
        f"forced_from_{prior_status}:{reason}" for reason in prior_reasons
    ] or ["forced_rewrite"]
    return record


def _finalize_sentence(

    record: SentenceRecord,

    *,

    lexical_enabled: bool,

    force_enabled: bool,

    paraphrase_enabled: bool,

    require_wording: bool,

    lexical_min_wsd: float,

    lexical_max_changes: int | None,

    lexical_polish: bool,

    safety_min: float,

    min_confidence: float,

    use_minilm: bool,

    variation_seed: int | None = None,

) -> SentenceRecord:
    # Paraphrase first as the main rewrite when enabled: pick a meaning-safe
    # but surface-divergent wording. Structural fallbacks cover gaps. Synonym
    # refinement then polishes the chosen rewrite.
    record = _apply_paraphrase_fallback(
        record,
        enabled=paraphrase_enabled,
        safety_min=safety_min,
        min_confidence=min_confidence,
        use_minilm=use_minilm,
        variation_seed=variation_seed,
    )
    record = _apply_forced_rewrite(record, enabled=force_enabled)
    record = _apply_phrase_rewrite(
        record,
        enabled=ENGINE_PHRASE_REWRITE,
        polish=lexical_polish,
        safety_min=safety_min,
        min_confidence=min_confidence,
        use_minilm=use_minilm,
    )
    record = _apply_lexical_refinement(
        record,
        enabled=lexical_enabled,
        min_wsd=lexical_min_wsd,
        max_changes=lexical_max_changes,
        safety_min=safety_min,
        min_confidence=min_confidence,
        use_minilm=use_minilm,
        polish=lexical_polish,
    )
    record = _apply_extra_polish(
        record,
        enabled=lexical_enabled and lexical_polish,
        min_wsd=lexical_min_wsd,
        max_changes=lexical_max_changes,
        safety_min=safety_min,
        min_confidence=min_confidence,
        use_minilm=use_minilm,
    )
    return _ensure_wording_changed(
        record,
        enabled=require_wording,
        lexical_min_wsd=lexical_min_wsd,
        lexical_max_changes=lexical_max_changes,
        lexical_polish=lexical_polish,
        safety_min=safety_min,
        min_confidence=min_confidence,
        use_minilm=use_minilm,
    )


def _ensure_wording_changed(

    record: SentenceRecord,

    *,

    enabled: bool,

    lexical_min_wsd: float,

    lexical_max_changes: int | None,

    lexical_polish: bool,

    safety_min: float,

    min_confidence: float,

    use_minilm: bool,

) -> SentenceRecord:
    """Guarantee a meaning-safe wording change when the sentence is still identical."""
    if not enabled or record.sentence_type not in _REWRITEABLE_TYPES:
        return record
    if not _unchanged(record):
        return record

    # Voice shift is a structural last resort before synonym forcing.
    passive = active_to_passive(record.original)
    if passive:
        accepted = _try_accept_rewrite(
            record,
            passive,
            template_id="ensure_passive",
            confidence=0.62,
            safety_min=safety_min,
            min_confidence=min_confidence,
            use_minilm=use_minilm,
            reason_prefix="ensure_passive",
        )
        if accepted is not None:
            return accepted

    ensured = ensure_wording_change(
        record.original,
        min_wsd=min(lexical_min_wsd, 0.10),
        max_changes=lexical_max_changes,
        polish=lexical_polish,
        classical_strict=(not use_minilm),
        classical_aggressive=(not use_minilm) and ENGINE_CLASSICAL_AGGRESSIVE,
    )
    if not ensured.changes or ensured.text == record.original:
        return record
    if not sufficiently_changed(record.original, ensured.text):
        return record
    safety = check_safety(
        record.original,
        ensured.text,
        min_meaning=min(safety_min, ENGINE_PARAPHRASE_MIN_SIM),
        min_confidence=min_confidence,
        use_minilm=use_minilm,
        protected_entities=None,
        protected_auxiliaries=None,
        structural_validation=False,
    )
    if not safety.ok:
        hard = check_safety(
            record.original,
            ensured.text,
            hard_invariants_only=True,
        )
        if not hard.ok:
            return record
        safety = hard

    prior_status = record.status
    prior_reasons = list(record.reasons)
    record.rewritten = ensured.text
    record.lexical_changes = ensured.changes
    record.status = "rewritten"
    record.template_id = "ensure_wording"
    record.confidence = min(0.70, max(0.55, safety.confidence, ensured.confidence))
    record.reasons = [
        f"ensure_from_{prior_status}:{reason}" for reason in prior_reasons
    ] or ["ensure_wording_change"]
    return record


def _process_sentence(

    text: str,

    *,

    index: int,

    block_index: int,

    min_confidence: float,

    safety_min: float,

    use_minilm: bool,

    lexical_enabled: bool,

    force_enabled: bool,

    paraphrase_enabled: bool,

    require_wording: bool,

    lexical_min_wsd: float,

    lexical_max_changes: int | None,

    lexical_polish: bool,

    last_template: str,

    stats: EngineStats,

    variation_seed: int | None = None,

) -> tuple[SentenceRecord, str]:
    """Rewrite one sentence with plan → generate → grammar → safety → fallback."""
    original = (text or "").strip()
    kind = classify_sentence(original)

    if not original:
        rec = SentenceRecord(
            index=index,
            original=original,
            rewritten=original,
            confidence=1.0,
            status="passthrough",
            sentence_type=kind,
            block_index=block_index,
        )
        stats.passthrough += 1
        return rec, ""

    plan = build_plan(original, min_confidence=min_confidence)
    if not plan.safe:
        reason = plan.skip_reason or kind
        stats.skipped += 1
        stats.bump(reason)
        rec = SentenceRecord(
            index=index,
            original=original,
            rewritten=original,
            confidence=plan.confidence,
            status="skipped",
            sentence_type=kind,
            reasons=[reason],
            block_index=block_index,
        )
        rec = _finalize_sentence(
            rec,
            lexical_enabled=lexical_enabled,
            force_enabled=force_enabled,
            paraphrase_enabled=paraphrase_enabled,
            require_wording=require_wording,
            lexical_min_wsd=lexical_min_wsd,
            lexical_max_changes=lexical_max_changes,
            lexical_polish=lexical_polish,
            safety_min=safety_min,
            min_confidence=min_confidence,
            use_minilm=use_minilm,
            variation_seed=variation_seed,
        )
        return rec, rec.template_id if rec.status == "rewritten" else ""

    # Prefer alternate template if last sentence used the same time-front family
    time_fronts = {
        "time_subj_manner_verb_place",
        "time_subj_verb_place",
        "time_subj_verb_object",
        "time_subj_verb_object_place",
        "time_front",
    }
    # Slot templates alone rarely offer more than one survivor, so fold in the
    # clause/voice/split transforms. Rotation then has real alternatives to pick
    # between instead of always landing on the same fronting.
    pool: list[tuple[str, str | None, float]] = [
        (cand.template_id, None, cand.confidence) for cand in plan.candidates
    ]
    if variation_seed is not None:
        planned = {template_id for template_id, _text, _conf in pool}
        pool += [
            (template_id, text, confidence)
            for template_id, text, confidence in structural_fallback_candidates(
                original, include_plan=False
            )
            # Passivizing reads worse than any reorder, so it stays a last
            # resort in the fallback path rather than a rotation option.
            if template_id not in planned and template_id != "active_to_passive"
        ]

    candidates = rotate(
        pool,
        seed=variation_seed,
        position=index,
        confidence_of=lambda item: item[2],
    )
    if last_template in time_fronts and candidates:
        reordered = [c for c in candidates if c[0] != last_template]
        reordered += [c for c in candidates if c[0] == last_template]
        candidates = reordered

    best: SentenceRecord | None = None
    for template_id, prepared, cand_confidence in candidates:
        generated = (
            prepared
            if prepared is not None
            else generate_from_plan(plan, template_id=template_id)
        )
        if not generated:
            continue
        # Voice flips legitimately move the agent and rebuild the auxiliary
        # chain, so slot-level protections do not apply to them.
        voice_flip = template_id in {"active_to_passive", "passive_to_active"}
        safety = check_safety(
            original,
            generated,
            min_meaning=safety_min,
            min_confidence=min_confidence,
            use_minilm=use_minilm,
            protected_entities=(
                None if voice_flip else plan.slots.entities if plan.slots else ()
            ),
            protected_auxiliaries=(
                None if voice_flip else plan.slots.auxiliaries if plan.slots else ()
            ),
            structural_validation=not voice_flip,
        )
        if not safety.ok and (not use_minilm) and ENGINE_CLASSICAL_AGGRESSIVE:
            # Aggressive classical: accept structure when hard facts still hold.
            hard = check_safety(
                original,
                generated,
                hard_invariants_only=True,
            )
            if hard.ok:
                safety = hard
            else:
                stats.bump(safety.reasons[0] if safety.reasons else "safety")
                continue
        elif not safety.ok:
            stats.bump(safety.reasons[0] if safety.reasons else "safety")
            continue
        repaired = repair_sentence(generated) if GRAMMAR_FIX_OUTPUT else generated
        if repaired != generated:
            repaired_safety = check_safety(
                original,
                repaired,
                min_meaning=safety_min,
                min_confidence=min_confidence,
                use_minilm=use_minilm,
                protected_entities=(
                    None if voice_flip else plan.slots.entities if plan.slots else ()
                ),
                protected_auxiliaries=(
                    None
                    if voice_flip
                    else plan.slots.auxiliaries if plan.slots else ()
                ),
                structural_validation=False,
            )
            if not repaired_safety.ok:
                stats.bump(
                    repaired_safety.reasons[0]
                    if repaired_safety.reasons
                    else "grammar_safety"
                )
                continue
            safety = repaired_safety
        conf = min(cand_confidence, safety.confidence + 0.2)
        best = SentenceRecord(
            index=index,
            original=original,
            rewritten=repaired,
            confidence=conf,
            status="rewritten",
            template_id=template_id,
            sentence_type=kind,
            reasons=[],
            block_index=block_index,
        )
        break

    if best is None:
        stats.reverted += 1
        stats.bump("safety_fallback")
        rec = SentenceRecord(
            index=index,
            original=original,
            rewritten=original,
            confidence=0.0,
            status="reverted",
            sentence_type=kind,
            reasons=["safety_fallback"],
            block_index=block_index,
        )
        rec = _finalize_sentence(
            rec,
            lexical_enabled=lexical_enabled,
            force_enabled=force_enabled,
            paraphrase_enabled=paraphrase_enabled,
            require_wording=require_wording,
            lexical_min_wsd=lexical_min_wsd,
            lexical_max_changes=lexical_max_changes,
            lexical_polish=lexical_polish,
            safety_min=safety_min,
            min_confidence=min_confidence,
            use_minilm=use_minilm,
            variation_seed=variation_seed,
        )
        return rec, rec.template_id if rec.status == "rewritten" else ""

    stats.rewritten += 1
    best = _finalize_sentence(
        best,
        lexical_enabled=lexical_enabled,
        force_enabled=force_enabled,
        paraphrase_enabled=paraphrase_enabled,
        require_wording=require_wording,
        lexical_min_wsd=lexical_min_wsd,
        lexical_max_changes=lexical_max_changes,
        lexical_polish=lexical_polish,
        safety_min=safety_min,
        min_confidence=min_confidence,
        use_minilm=use_minilm,
        variation_seed=variation_seed,
    )
    return best, best.template_id


def _process_block(

    block: DocumentBlock,

    *,

    sent_offset: int,

    min_confidence: float,

    safety_min: float,

    use_minilm: bool,

    lexical_enabled: bool,

    force_enabled: bool,

    paraphrase_enabled: bool,

    require_wording: bool,

    lexical_min_wsd: float,

    lexical_max_changes: int | None,

    lexical_polish: bool,

    last_template: str,

    stats: EngineStats,

    variation_seed: int | None = None,

) -> tuple[DocumentBlock, list[SentenceRecord], str]:
    if not block.rewriteable or block.kind != "paragraph":
        stats.passthrough += 1
        rec = SentenceRecord(
            index=sent_offset,
            original=block.text,
            rewritten=block.text,
            confidence=1.0,
            status="passthrough",
            sentence_type=block.kind,
            block_index=block.index,
            reasons=[f"block:{block.kind}"],
        )
        return block, [rec], last_template

    masked, prot = mask_protected_spans(block.text)
    sentences = split_sentences(masked)
    if not sentences:
        return block, [], last_template

    records: list[SentenceRecord] = []
    out_sents: list[str] = []
    tid = last_template
    for i, sent in enumerate(sentences):
        rec, tid = _process_sentence(
            sent,
            index=sent_offset + i,
            block_index=block.index,
            min_confidence=min_confidence,
            safety_min=safety_min,
            use_minilm=use_minilm,
            lexical_enabled=lexical_enabled,
            force_enabled=force_enabled,
            paraphrase_enabled=paraphrase_enabled,
            require_wording=require_wording,
            lexical_min_wsd=lexical_min_wsd,
            lexical_max_changes=lexical_max_changes,
            lexical_polish=lexical_polish,
            last_template=tid,
            stats=stats,
            variation_seed=variation_seed,
        )
        # Unmask protected spans in both sides of the record
        rec.original = unmask_protected_spans(rec.original, prot)
        rec.rewritten = unmask_protected_spans(rec.rewritten, prot)
        records.append(rec)
        out_sents.append(rec.rewritten)

    new_text = unmask_protected_spans(join_sentences(out_sents), prot)
    new_block = DocumentBlock(
        text=new_text,
        kind=block.kind,
        rewriteable=block.rewriteable,
        index=block.index,
        meta=dict(block.meta),
    )
    return new_block, records, tid


def rewrite_document(

    source: str | bytes | None,

    *,

    batch_paras: int | None = None,

    min_confidence: float | None = None,

    safety_min: float | None = None,

    use_minilm_safety: bool | None = None,

    use_lexical_refinement: bool | None = None,

    force_rewrite: bool | None = None,

    use_paraphrase: bool | None = None,

    require_wording_change: bool | None = None,

    lexical_min_wsd: float | None = None,

    lexical_max_changes: int | None = None,

    lexical_polish: bool | None = None,

    preserve_length: bool | None = None,

    variation_seed: int | None = None,

) -> EngineResult:
    """

    Structural rewrite with local paraphrase and low-impact synonyms.



    Safe spaCy templates still run first. When paraphrase primary mode is on,

    a CPU T5 model then prefers a meaning-safe but surface-divergent rewrite so

    the text looks substantially new. MiniLM guards meaning. Synonym refinement

    runs after paraphrase. When require_wording_change is on, a final ensure

    pass forces a meaning-safe wording change whenever a safe option exists.



    Synonym budget is dynamic from sentence length unless lexical_max_changes

    is set explicitly. lexical_polish densifies that budget (UI "word polish").



    ``variation_seed`` rotates between equally-ranked structural options so the

    same text rewrites differently on each request. Pass an explicit seed for

    reproducible output; pass 0 to disable rotation entirely.

    """
    started = time.perf_counter()
    batch_paras = batch_paras if batch_paras is not None else ENGINE_BATCH_PARAS
    min_confidence = (
        min_confidence if min_confidence is not None else ENGINE_MIN_CONFIDENCE
    )
    safety_min = safety_min if safety_min is not None else ENGINE_SAFETY_MIN
    use_minilm = (
        use_minilm_safety
        if use_minilm_safety is not None
        else ENGINE_USE_MINILM_SAFETY
    )
    lexical_enabled = (
        use_lexical_refinement
        if use_lexical_refinement is not None
        else ENGINE_LEXICAL_REFINEMENT
    )
    force_enabled = (
        force_rewrite if force_rewrite is not None else ENGINE_FORCE_REWRITE
    )
    paraphrase_enabled = (
        use_paraphrase if use_paraphrase is not None else ENGINE_PARAPHRASE
    )
    require_wording = (
        require_wording_change
        if require_wording_change is not None
        else ENGINE_REQUIRE_WORDING_CHANGE
    )
    lexical_min_wsd = (
        lexical_min_wsd
        if lexical_min_wsd is not None
        else ENGINE_LEXICAL_MIN_WSD
    )
    # None => fully dynamic per-sentence budget inside refine_sentence.
    if lexical_max_changes is not None and lexical_max_changes <= 0:
        lexical_max_changes = None
    polish_enabled = bool(lexical_polish)
    keep_length = (
        ENGINE_PRESERVE_LENGTH if preserve_length is None else bool(preserve_length)
    )
    if variation_seed is None:
        variation_seed = new_seed() if ENGINE_STRUCTURAL_VARIATION else None
    elif variation_seed == 0:
        variation_seed = None
    if (force_enabled or paraphrase_enabled or require_wording) and min_confidence > 0.40:
        # Prefer more structural templates before generative fallback.
        min_confidence = 0.40

    raw = ingest_text(source)
    if not raw.strip():
        raise ValueError("Paste some text first.")

    normalized = normalize_text(raw)

    blocks = detect_blocks(normalized)
    stats = EngineStats(blocks=len(blocks))
    all_records: list[SentenceRecord] = []
    out_blocks: list[DocumentBlock] = []
    sent_offset = 0
    last_template = ""

    for batch in iter_paragraph_batches(blocks, batch_paras=batch_paras):
        stats.batches += 1
        for block in batch:
            new_block, records, last_template = _process_block(
                block,
                sent_offset=sent_offset,
                min_confidence=min_confidence,
                safety_min=safety_min,
                use_minilm=use_minilm,
                lexical_enabled=lexical_enabled,
                force_enabled=force_enabled,
                paraphrase_enabled=paraphrase_enabled,
                require_wording=require_wording,
                lexical_min_wsd=lexical_min_wsd,
                lexical_max_changes=lexical_max_changes,
                lexical_polish=polish_enabled,
                last_template=last_template,
                stats=stats,
                variation_seed=variation_seed,
            )
            out_blocks.append(new_block)
            all_records.extend(records)
            # Advance offset by sentence count for rewriteable paras
            if block.rewriteable and block.kind == "paragraph":
                sent_offset += max(1, len(records))
            else:
                sent_offset += 1

    stats.sentences = len(all_records)
    stitched = stitch_blocks(out_blocks)

    # Consistency: suppress streaks then rebuild from records if needed
    stitched, all_records = apply_consistency(stitched, all_records)
    streak_reverts = [r for r in all_records if "template_streak" in r.reasons]
    if streak_reverts:
        # Rebuild rewriteable paragraph texts from records
        by_block: dict[int, list[SentenceRecord]] = {}
        for rec in all_records:
            by_block.setdefault(rec.block_index, []).append(rec)
        rebuilt: list[DocumentBlock] = []
        for block in out_blocks:
            recs = by_block.get(block.index, [])
            if block.rewriteable and block.kind == "paragraph" and recs:
                text = join_sentences([r.rewritten for r in recs])
                rebuilt.append(
                    DocumentBlock(
                        text=text,
                        kind=block.kind,
                        rewriteable=True,
                        index=block.index,
                    )
                )
            else:
                rebuilt.append(block)
        stitched = stitch_blocks(rebuilt)
        stitched, all_records = apply_consistency(stitched, all_records)

    stitched = tidy(stitched)
    if keep_length:
        trimmed = enforce_length_budget(
            normalized,
            stitched,
            preserve_length=True,
        )
        if trimmed != stitched:
            stitched = tidy(trimmed)
            notes_trim = "length_budget"
        else:
            notes_trim = ""
    else:
        notes_trim = ""

    stats.rewritten = sum(r.status == "rewritten" for r in all_records)
    stats.skipped = sum(r.status == "skipped" for r in all_records)
    stats.reverted = sum(r.status == "reverted" for r in all_records)
    stats.passthrough = sum(r.status == "passthrough" for r in all_records)
    stats.lexical_refined = sum(bool(r.lexical_changes) for r in all_records)
    stats.forced_rewrites = sum(
        (r.template_id or "").startswith("forced_cleft") for r in all_records
    )
    stats.paraphrased = sum(
        (r.template_id or "").startswith("paraphrase")
        or any(
            reason.startswith("structure_fallback") for reason in (r.reasons or [])
        )
        for r in all_records
    )
    stats.seconds = round(time.perf_counter() - started, 3)
    mapping = [(r.original, r.rewritten) for r in all_records]
    skipped = [r for r in all_records if r.status in {"skipped", "reverted"}]
    in_words = word_count(normalized)
    out_words = word_count(stitched)
    changed = stitched.strip() != normalized.strip()
    sim = _similarity(normalized, stitched)

    notes = (
        f"structural: rewritten={stats.rewritten} skipped={stats.skipped} "
        f"reverted={stats.reverted} lexical={stats.lexical_refined} "
        f"paraphrased={stats.paraphrased} forced={stats.forced_rewrites} "
        f"batches={stats.batches}"
    )
    if notes_trim:
        notes = f"{notes}; {notes_trim}"
    logger.info(
        "engine done words=%s→%s rewritten=%s skipped=%s reverted=%s "
        "paraphrased=%s lexical=%s batches=%s t=%.2fs",
        in_words,
        out_words,
        stats.rewritten,
        stats.skipped,
        stats.reverted,
        stats.paraphrased,
        stats.lexical_refined,
        stats.batches,
        stats.seconds,
    )

    return EngineResult(
        text=stitched,
        sentences=all_records,
        skipped=skipped,
        mapping=mapping,
        stats=stats,
        notes=notes,
        engine="structural-reorder",
        input_words=in_words,
        output_words=out_words,
        changed=changed,
        similarity=round(sim, 4),
    )