Upload 142 files

app/bootstrap.py

@@ -1,4 +1,4 @@
-"""Bootstrap spaCy + NLTK WordNet data (used by Docker entrypoint and local first run)."""
+"""Bootstrap spaCy (used by Docker entrypoint and local first run)."""
 
 from __future__ import annotations
 
@@ -12,7 +12,6 @@ def ensure_resources() -> None:
         try:
             spacy.load(SPACY_MODEL)
         except OSError:
-            # Prefer wheel install in Docker; CLI download can fail on SSL.
             print(
                 f"[warn] spaCy model '{SPACY_MODEL}' not installed. "
                 "Using regex fallback. Install with: "
@@ -23,30 +22,15 @@ def ensure_resources() -> None:
         print(f"[warn] spaCy setup: {exc}")
 
     try:
-        import nltk
-        from nltk.corpus import wordnet as wn
+        from app.config import ENGINE_USE_MINILM_SAFETY, ML_POLISH_WARM
 
-        try:
-            wn.synsets("test")
-        except LookupError:
-            nltk.download("wordnet", quiet=True)
-            nltk.download("omw-1.4", quiet=True)
-    except Exception as exc:  # noqa: BLE001
-        print(f"[warn] NLTK WordNet setup: {exc}")
-
-    try:
-        from app.config import ML_POLISH_WARM
-
-        if ML_POLISH_WARM:
-            from app.pipeline.generative import warm_generative
+        if ML_POLISH_WARM or ENGINE_USE_MINILM_SAFETY:
             from app.pipeline.minilm import warm_minilm
 
             ok_m = warm_minilm()
             print(f"[info] MiniLM warm: {'ok' if ok_m else 'skipped/unavailable'}")
-            ok_g = warm_generative()
-            print(f"[info] Generative warm: {'ok' if ok_g else 'skipped/unavailable'}")
     except Exception as exc:  # noqa: BLE001
-        print(f"[warn] ML polish warm: {exc}")
+        print(f"[warn] MiniLM warm: {exc}")
 
 
 if __name__ == "__main__":

app/config.py

@@ -73,14 +73,16 @@ if _gen_backend_raw in {"causal", "chat", "instruct", "decoder"}:
     GENERATIVE_BACKEND = "causal"
 else:
     GENERATIVE_BACKEND = "seq2seq"
-# Pipeline: hybrid | generative | classical
-_pipe_raw = (os.environ.get("PIPELINE_MODE") or "hybrid").strip().lower()
+# Pipeline: structural (default) | hybrid | generative | classical (legacy)
+_pipe_raw = (os.environ.get("PIPELINE_MODE") or "structural").strip().lower()
 if _pipe_raw in {"generative", "gen", "llm"}:
     PIPELINE_MODE = "generative"
 elif _pipe_raw in {"classical", "rules", "lexicon"}:
     PIPELINE_MODE = "classical"
-else:
+elif _pipe_raw in {"hybrid"}:
     PIPELINE_MODE = "hybrid"
+else:
+    PIPELINE_MODE = "structural"
 # 0 = rewrite entire document (no paragraph cap). Positive = safety cap.
 _gen_max_raw = (
     os.environ.get("GENERATIVE_MAX_PARAGRAPHS")
@@ -112,5 +114,22 @@ _gfo = (os.environ.get("GRAMMAR_FIX_OUTPUT") or "true").strip().lower()
 GRAMMAR_FIX_OUTPUT = _gfo not in {"0", "false", "no", "off"}
 
 # Classical lexicon path: only used when generative is unavailable / ML polish off
-_lex = (os.environ.get("LEXICON_FALLBACK") or "true").strip().lower()
+_lex = (os.environ.get("LEXICON_FALLBACK") or "false").strip().lower()
 LEXICON_FALLBACK = _lex not in {"0", "false", "no", "off"}
+
+# ---------------------------------------------------------------------------
+# Structural rewrite engine (primary path)
+# ---------------------------------------------------------------------------
+ENGINE_BATCH_PARAS = max(
+    1, min(int(os.environ.get("ENGINE_BATCH_PARAS", "20") or "20"), 100)
+)
+ENGINE_MIN_CONFIDENCE = max(
+    0.0, min(float(os.environ.get("ENGINE_MIN_CONFIDENCE", "0.55") or "0.55"), 1.0)
+)
+ENGINE_SAFETY_MIN = max(
+    0.0, min(float(os.environ.get("ENGINE_SAFETY_MIN", "0.80") or "0.80"), 1.0)
+)
+_syn = (os.environ.get("ALLOW_SYNONYM_REPLACEMENT") or "false").strip().lower()
+ALLOW_SYNONYM_REPLACEMENT = _syn in {"1", "true", "yes", "on"}
+_ems = (os.environ.get("ENGINE_USE_MINILM_SAFETY") or "false").strip().lower()
+ENGINE_USE_MINILM_SAFETY = _ems in {"1", "true", "yes", "on"}

app/engine/__init__.py

@@ -0,0 +1,6 @@
+"""CPU-friendly rule-based document rewrite engine (structural reorder)."""
+
+from app.engine.models import EngineResult, SentenceRecord
+from app.engine.orchestrator import rewrite_document
+
+__all__ = ["rewrite_document", "EngineResult", "SentenceRecord"]

app/engine/classify/__init__.py

@@ -0,0 +1,62 @@
+"""Sentence type classification — only safe types enter the rewrite path."""
+
+from __future__ import annotations
+
+import re
+
+_SUBORDINATORS = frozenset(
+    """
+    although though while whilst whereas unless until since
+    if when whenever wherever whether before after
+    """.split()
+)
+
+_REWRITEABLE = frozenset({"simple_declarative", "compound", "because_clause"})
+
+
+def classify_sentence(text: str) -> str:
+    """Tag sentence type for rewrite eligibility."""
+    t = (text or "").strip()
+    if not t:
+        return "empty"
+    if re.match(r"^#{1,6}\s", t) or (len(t.split()) <= 6 and t.isupper()):
+        return "heading"
+    if re.match(r"^(\d+[\.\)]\s+|[-*•]\s+)", t):
+        return "list_item"
+    if t.startswith(('"', "'", "\u201c", "\u2018")) and (
+        t.count('"') >= 2 or t.count("\u201c") or t.count("'") >= 2
+    ):
+        return "quoted"
+    if "?" in t or t.endswith("?"):
+        return "question"
+    if len(t.split()) < 3:
+        return "too_short"
+    if len(t.split()) > 45:
+        return "too_long"
+
+    low = t.lower()
+    if re.search(r"\bbecause\b", low):
+        return "because_clause"
+
+    # Coordinating compound (safe if short enough)
+    if re.search(r"\b(and|but|or|so|yet)\b", low) and "," in t:
+        # Still complex if subordinators present
+        for sub in _SUBORDINATORS:
+            if re.search(rf"\b{re.escape(sub)}\b", low):
+                return "complex"
+        if len(t.split()) <= 28:
+            return "compound"
+        return "complex"
+
+    for sub in _SUBORDINATORS:
+        if re.search(rf"\b{re.escape(sub)}\b", low):
+            return "complex"
+
+    if re.search(r"\b(who|whom|whose|which|that)\b", low) and len(t.split()) > 12:
+        return "complex"
+
+    return "simple_declarative"
+
+
+def is_rewriteable_type(sentence_type: str) -> bool:
+    return sentence_type in _REWRITEABLE

app/engine/consistency/__init__.py

@@ -0,0 +1,94 @@
+"""Document-level consistency pass after sentence rewrite."""
+
+from __future__ import annotations
+
+import re
+from collections import defaultdict
+
+from app.engine.models import SentenceRecord
+
+
+def _proper_nouns(text: str) -> list[str]:
+    return re.findall(r"\b[A-Z][a-zA-Z0-9'-]+\b", text or "")
+
+
+def canonicalize_entities(text: str, records: list[SentenceRecord]) -> str:
+    """Normalize entity spelling to the first-seen form across the document."""
+    # Build map: lowercase → canonical first spelling
+    canon: dict[str, str] = {}
+    for rec in records:
+        for name in _proper_nouns(rec.original):
+            key = name.lower()
+            if key not in canon:
+                canon[key] = name
+
+    if not canon:
+        return text
+
+    def repl(m: re.Match[str]) -> str:
+        word = m.group(0)
+        key = word.lower()
+        if key in canon and word != canon[key]:
+            # Only fix case variants of same spelling length
+            if word.lower() == canon[key].lower():
+                return canon[key]
+        return word
+
+    return re.sub(r"\b[A-Za-z][A-Za-z0-9'-]*\b", repl, text)
+
+
+def suppress_template_streaks(records: list[SentenceRecord]) -> list[SentenceRecord]:
+    """If two consecutive rewrites used the same time-front template, revert the second."""
+    time_fronts = {
+        "time_subj_manner_verb_place",
+        "time_subj_verb_place",
+        "time_subj_verb_object",
+        "time_subj_verb_object_place",
+        "time_front",
+    }
+    out: list[SentenceRecord] = []
+    prev_tid = ""
+    for rec in records:
+        if (
+            rec.status == "rewritten"
+            and rec.template_id in time_fronts
+            and prev_tid == rec.template_id
+        ):
+            # Revert to original to avoid mechanical repetition
+            reverted = SentenceRecord(
+                index=rec.index,
+                original=rec.original,
+                rewritten=rec.original,
+                confidence=rec.confidence,
+                status="reverted",
+                template_id="",
+                sentence_type=rec.sentence_type,
+                reasons=rec.reasons + ["template_streak"],
+                block_index=rec.block_index,
+            )
+            out.append(reverted)
+            prev_tid = ""
+            continue
+        out.append(rec)
+        prev_tid = rec.template_id if rec.status == "rewritten" else ""
+    return out
+
+
+def apply_consistency(
+    text: str,
+    records: list[SentenceRecord],
+) -> tuple[str, list[SentenceRecord]]:
+    """Run light document-level consistency fixes."""
+    adjusted = suppress_template_streaks(records)
+    # Rebuild is caller's job when records change; here we only fix entity spelling on text
+    # if no streak reverts. When streaks revert, orchestrator should restitch.
+    fixed_text = canonicalize_entities(text, adjusted)
+    return fixed_text, adjusted
+
+
+def entity_frequency(records: list[SentenceRecord]) -> dict[str, int]:
+    freq: dict[str, int] = defaultdict(int)
+    for rec in records:
+        for name in _proper_nouns(rec.original):
+            freq[name] += 1
+    return dict(freq)

app/engine/grammar/__init__.py

@@ -0,0 +1,31 @@
+"""Sentence-scoped grammar repair after structural rewrite."""
+
+from __future__ import annotations
+
+import re
+
+from app.pipeline.grammar_fix import correct_text
+
+
+def repair_sentence(text: str) -> str:
+    """Fix punctuation, capitalization, agreement, spacing on one sentence."""
+    raw = (text or "").strip()
+    if not raw:
+        return raw
+    # Lightweight local cleanup before LanguageTool/spaCy agreement
+    s = re.sub(r"\s+", " ", raw).strip()
+    s = re.sub(r"\s+([,.;:!?])", r"\1", s)
+    s = re.sub(r",\s*,+", ",", s)
+    s = re.sub(r"\s+,", ",", s)
+    if s and s[0].islower():
+        s = s[0].upper() + s[1:]
+    try:
+        fixed = correct_text(s)
+        if fixed and fixed.strip():
+            # Prefer sentence-sized result; correct_text may expand oddly
+            out = fixed.strip()
+            if abs(len(out.split()) - len(s.split())) <= max(3, len(s.split()) // 2):
+                return out
+    except Exception:
+        pass
+    return s

app/engine/ingest/__init__.py

@@ -0,0 +1,17 @@
+"""Document ingest — raw text / bytes → UTF-8 string."""
+
+from __future__ import annotations
+
+
+def ingest_text(source: str | bytes | None, *, encoding: str = "utf-8") -> str:
+    """Normalize input into a UTF-8 Unicode string."""
+    if source is None:
+        return ""
+    if isinstance(source, bytes):
+        for enc in (encoding, "utf-8", "utf-8-sig", "latin-1"):
+            try:
+                return source.decode(enc)
+            except (UnicodeDecodeError, LookupError):
+                continue
+        return source.decode("utf-8", errors="replace")
+    return str(source)

app/engine/models.py

@@ -0,0 +1,104 @@
+"""Shared data models for the structural rewrite engine."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class DocumentBlock:
+    """A contiguous document region with rewrite policy."""
+
+    text: str
+    kind: str = "paragraph"  # paragraph|heading|list|table|code|formula|bibliography|blank
+    rewriteable: bool = True
+    index: int = 0
+    meta: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class SentenceSlots:
+    """Constituent spans extracted for template fill."""
+
+    text: str
+    subject: str = ""
+    verb: str = ""
+    verb_phrase: str = ""
+    object: str = ""
+    place: str = ""
+    time: str = ""
+    manner: str = ""
+    negation: str = ""
+    leftover: str = ""
+    entities: list[str] = field(default_factory=list)
+    confidence: float = 0.0
+    sentence_type: str = "unsupported"
+    reasons: list[str] = field(default_factory=list)
+
+
+@dataclass
+class TemplateCandidate:
+    template_id: str
+    confidence: float
+
+
+@dataclass
+class RewritePlan:
+    """Decision object before generation."""
+
+    safe: bool
+    slots: SentenceSlots | None = None
+    template_id: str = ""
+    candidates: list[TemplateCandidate] = field(default_factory=list)
+    fixed_spans: list[str] = field(default_factory=list)
+    movable: list[str] = field(default_factory=list)
+    skip_reason: str = ""
+    confidence: float = 0.0
+
+
+@dataclass
+class SentenceRecord:
+    """Per-sentence rewrite report."""
+
+    index: int
+    original: str
+    rewritten: str
+    confidence: float
+    status: str  # rewritten|skipped|reverted|passthrough
+    template_id: str = ""
+    sentence_type: str = ""
+    reasons: list[str] = field(default_factory=list)
+    block_index: int = 0
+
+
+@dataclass
+class EngineStats:
+    batches: int = 0
+    blocks: int = 0
+    sentences: int = 0
+    rewritten: int = 0
+    skipped: int = 0
+    reverted: int = 0
+    passthrough: int = 0
+    seconds: float = 0.0
+    reasons: dict[str, int] = field(default_factory=dict)
+
+    def bump(self, reason: str) -> None:
+        key = (reason or "other").split(":")[0]
+        self.reasons[key] = self.reasons.get(key, 0) + 1
+
+
+@dataclass
+class EngineResult:
+    text: str
+    sentences: list[SentenceRecord] = field(default_factory=list)
+    skipped: list[SentenceRecord] = field(default_factory=list)
+    mapping: list[tuple[str, str]] = field(default_factory=list)
+    stats: EngineStats = field(default_factory=EngineStats)
+    notes: str = ""
+    engine: str = "structural-reorder"
+    input_words: int = 0
+    output_words: int = 0
+    changed: bool = False
+    similarity: float = 1.0

app/engine/normalize/__init__.py

@@ -0,0 +1,258 @@
+"""Cleaning, punctuation normalization, and block tagging."""
+
+from __future__ import annotations
+
+import re
+
+from app.engine.models import DocumentBlock
+
+_BIBLIO_HEADINGS = re.compile(
+    r"^(#{1,6}\s*)?(references|bibliography|works\s+cited|citations)\s*:?\s*$",
+    re.I | re.M,
+)
+_HEADING = re.compile(r"^(#{1,6}\s+\S.*|[A-Z][A-Z0-9 ,.'-]{2,60})$")
+_LIST_ITEM = re.compile(r"^(\d+[\.\)]\s+|[-*•]\s+)\S")
+_CODE_FENCE = re.compile(r"^```")
+_TABLE_LINE = re.compile(r"^\s*\|.+\|\s*$")
+_HTML_TABLE = re.compile(r"</?table\b", re.I)
+_FORMULA = re.compile(r"(\$\$.+?\$\$|\\\[[\s\S]+?\\\]|\\begin\{(?:equation|align|math)\})")
+_URL = re.compile(r"https?://[^\s<>\"']+|www\.[^\s<>\"']+", re.I)
+_EMAIL = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b")
+_SOFT_WRAP = re.compile(r"(?<=\w)\n(?=\w)")
+
+
+def normalize_text(text: str) -> str:
+    """UTF-8 hygiene, quotes/dashes, soft wraps, spacing."""
+    t = (text or "").replace("\r\n", "\n").replace("\r", "\n")
+    # Smart quotes / dashes → ASCII equivalents
+    t = (
+        t.replace("\u201c", '"')
+        .replace("\u201d", '"')
+        .replace("\u2018", "'")
+        .replace("\u2019", "'")
+        .replace("\u2013", "-")
+        .replace("\u2014", "—")
+        .replace("\u00a0", " ")
+    )
+    # Fix broken line wraps inside paragraphs (keep blank-line paragraph breaks)
+    parts = re.split(r"(\n\s*\n)", t)
+    fixed: list[str] = []
+    for part in parts:
+        if re.match(r"\n\s*\n", part):
+            fixed.append("\n\n")
+            continue
+        # Don't join wraps inside code fences
+        if "```" in part:
+            fixed.append(part)
+            continue
+        lines = part.split("\n")
+        rebuilt: list[str] = []
+        buf = ""
+        for line in lines:
+            stripped = line.strip()
+            if not stripped:
+                if buf:
+                    rebuilt.append(buf)
+                    buf = ""
+                rebuilt.append("")
+                continue
+            if _LIST_ITEM.match(stripped) or _HEADING.match(stripped) or _TABLE_LINE.match(stripped):
+                if buf:
+                    rebuilt.append(buf)
+                    buf = ""
+                rebuilt.append(stripped)
+                continue
+            if buf and not buf.endswith(("-", "—")):
+                # Soft wrap: join with space
+                buf = f"{buf} {stripped}"
+            elif buf and buf.endswith(("-", "—")):
+                buf = buf.rstrip("-—") + stripped
+            else:
+                buf = stripped
+        if buf:
+            rebuilt.append(buf)
+        fixed.append("\n".join(rebuilt))
+    t = "".join(fixed)
+    t = re.sub(r"[ \t]+", " ", t)
+    t = re.sub(r" *\n *", "\n", t)
+    t = re.sub(r"\n{3,}", "\n\n", t)
+    # Punctuation spacing
+    t = re.sub(r"\s+([,.;:!?])", r"\1", t)
+    t = re.sub(r"([(\[{])\s+", r"\1", t)
+    t = re.sub(r"\s+([)\]}])", r"\1", t)
+    return t.strip()
+
+
+def _is_bibliography_heading(line: str) -> bool:
+    return bool(_BIBLIO_HEADINGS.match(line.strip()))
+
+
+def detect_blocks(text: str) -> list[DocumentBlock]:
+    """Split normalized text into typed blocks with rewrite flags."""
+    raw = text or ""
+    if not raw.strip():
+        return []
+
+    blocks: list[DocumentBlock] = []
+    chunks = re.split(r"(\n\s*\n)", raw)
+    in_code = False
+    in_biblio = False
+    buf_lines: list[str] = []
+    buf_kind = "paragraph"
+    index = 0
+
+    def flush() -> None:
+        nonlocal index, buf_lines, buf_kind
+        if not buf_lines:
+            return
+        body = "\n".join(buf_lines).strip("\n")
+        if body.strip() == "" and body:
+            blocks.append(
+                DocumentBlock(text=body, kind="blank", rewriteable=False, index=index)
+            )
+        elif body.strip():
+            rewriteable = buf_kind == "paragraph" and not in_biblio
+            blocks.append(
+                DocumentBlock(
+                    text=body.strip(),
+                    kind=buf_kind if not in_biblio or buf_kind != "paragraph" else "bibliography",
+                    rewriteable=rewriteable and buf_kind == "paragraph",
+                    index=index,
+                )
+            )
+        index += 1
+        buf_lines = []
+        buf_kind = "paragraph"
+
+    for chunk in chunks:
+        if re.match(r"\n\s*\n", chunk or ""):
+            flush()
+            blocks.append(
+                DocumentBlock(text="", kind="blank", rewriteable=False, index=index)
+            )
+            index += 1
+            continue
+
+        for line in (chunk or "").split("\n"):
+            stripped = line.strip()
+
+            if _CODE_FENCE.match(stripped):
+                if in_code:
+                    buf_lines.append(line)
+                    buf_kind = "code"
+                    flush()
+                    in_code = False
+                else:
+                    flush()
+                    in_code = True
+                    buf_kind = "code"
+                    buf_lines = [line]
+                continue
+
+            if in_code:
+                buf_lines.append(line)
+                buf_kind = "code"
+                continue
+
+            if _is_bibliography_heading(stripped):
+                flush()
+                in_biblio = True
+                buf_kind = "bibliography"
+                buf_lines = [stripped]
+                flush()
+                continue
+
+            if _HTML_TABLE.search(stripped) or _TABLE_LINE.match(stripped):
+                flush()
+                blocks.append(
+                    DocumentBlock(
+                        text=stripped, kind="table", rewriteable=False, index=index
+                    )
+                )
+                index += 1
+                continue
+
+            if _FORMULA.search(stripped) and len(stripped.split()) < 40:
+                flush()
+                blocks.append(
+                    DocumentBlock(
+                        text=stripped, kind="formula", rewriteable=False, index=index
+                    )
+                )
+                index += 1
+                continue
+
+            if _LIST_ITEM.match(stripped):
+                flush()
+                blocks.append(
+                    DocumentBlock(
+                        text=stripped, kind="list", rewriteable=False, index=index
+                    )
+                )
+                index += 1
+                continue
+
+            if _HEADING.match(stripped) and len(stripped.split()) <= 12:
+                flush()
+                blocks.append(
+                    DocumentBlock(
+                        text=stripped, kind="heading", rewriteable=False, index=index
+                    )
+                )
+                index += 1
+                continue
+
+            if in_biblio:
+                buf_kind = "bibliography"
+            else:
+                buf_kind = "paragraph"
+            buf_lines.append(stripped if stripped else line)
+
+    flush()
+
+    # Protect paragraphs that are mostly URL/email-only
+    for b in blocks:
+        if not b.rewriteable:
+            continue
+        plain = _URL.sub("", b.text)
+        plain = _EMAIL.sub("", plain).strip()
+        if len(plain.split()) < 3 and (_URL.search(b.text) or _EMAIL.search(b.text)):
+            b.rewriteable = False
+            b.kind = "special"
+
+    return blocks
+
+
+def mask_protected_spans(text: str) -> tuple[str, dict[str, str]]:
+    """Replace URLs/emails/paths with placeholders for rewrite safety."""
+    mapping: dict[str, str] = {}
+    counter = {"n": 0}
+
+    def _sub(pattern: re.Pattern[str], label: str, s: str) -> str:
+        def repl(m: re.Match[str]) -> str:
+            key = f"__PROT_{label}_{counter['n']}__"
+            mapping[key] = m.group(0)
+            counter["n"] += 1
+            return key
+
+        return pattern.sub(repl, s)
+
+    out = text
+    out = _sub(_URL, "URL", out)
+    out = _sub(_EMAIL, "EMAIL", out)
+
+    def path_repl(m: re.Match[str]) -> str:
+        key = f"__PROT_PATH_{counter['n']}__"
+        mapping[key] = m.group(0)
+        counter["n"] += 1
+        return key
+
+    out = re.sub(r"(?:[A-Za-z]:\\|/)[^\s<>\"']+", path_repl, out)
+    return out, mapping
+
+
+def unmask_protected_spans(text: str, mapping: dict[str, str]) -> str:
+    out = text
+    for key, val in mapping.items():
+        out = out.replace(key, val)
+    return out

app/engine/orchestrator.py

@@ -0,0 +1,354 @@
+"""Batch/stream orchestrator for the structural rewrite engine."""
+
+from __future__ import annotations
+
+import logging
+import time
+from difflib import SequenceMatcher
+
+from app.config import (
+    ALLOW_SYNONYM_REPLACEMENT,
+    ENGINE_BATCH_PARAS,
+    ENGINE_MIN_CONFIDENCE,
+    ENGINE_SAFETY_MIN,
+    ENGINE_USE_MINILM_SAFETY,
+    GRAMMAR_FIX_INPUT,
+    GRAMMAR_FIX_OUTPUT,
+)
+from app.engine.classify import classify_sentence
+from app.engine.consistency import apply_consistency
+from app.engine.grammar import repair_sentence
+from app.engine.ingest import ingest_text
+from app.engine.models import (
+    DocumentBlock,
+    EngineResult,
+    EngineStats,
+    SentenceRecord,
+)
+from app.engine.normalize import (
+    detect_blocks,
+    mask_protected_spans,
+    normalize_text,
+    unmask_protected_spans,
+)
+from app.engine.plan import build_plan
+from app.engine.rewrite import generate_from_plan
+from app.engine.safety import check_safety
+from app.engine.segment import iter_paragraph_batches, split_sentences, word_count
+from app.engine.stitch import join_sentences, stitch_blocks
+
+logger = logging.getLogger("plainrewrite.engine")
+
+# Synonym path is intentionally unused unless explicitly enabled (not implemented on hot path).
+_ = ALLOW_SYNONYM_REPLACEMENT
+
+
+def _similarity(a: str, b: str) -> float:
+    return SequenceMatcher(None, (a or "").lower(), (b or "").lower()).ratio()
+
+
+def _process_sentence(
+    text: str,
+    *,
+    index: int,
+    block_index: int,
+    min_confidence: float,
+    safety_min: float,
+    use_minilm: bool,
+    last_template: str,
+    stats: EngineStats,
+) -> tuple[SentenceRecord, str]:
+    """Rewrite one sentence with plan → generate → grammar → safety → fallback."""
+    original = (text or "").strip()
+    kind = classify_sentence(original)
+
+    if not original:
+        rec = SentenceRecord(
+            index=index,
+            original=original,
+            rewritten=original,
+            confidence=1.0,
+            status="passthrough",
+            sentence_type=kind,
+            block_index=block_index,
+        )
+        stats.passthrough += 1
+        return rec, ""
+
+    plan = build_plan(original, min_confidence=min_confidence)
+    if not plan.safe:
+        reason = plan.skip_reason or kind
+        stats.skipped += 1
+        stats.bump(reason)
+        return (
+            SentenceRecord(
+                index=index,
+                original=original,
+                rewritten=original,
+                confidence=plan.confidence,
+                status="skipped",
+                sentence_type=kind,
+                reasons=[reason],
+                block_index=block_index,
+            ),
+            "",
+        )
+
+    # Prefer alternate template if last sentence used the same time-front family
+    time_fronts = {
+        "time_subj_manner_verb_place",
+        "time_subj_verb_place",
+        "time_subj_verb_object",
+        "time_subj_verb_object_place",
+        "time_front",
+    }
+    candidates = list(plan.candidates)
+    if last_template in time_fronts and candidates:
+        reordered = [c for c in candidates if c.template_id != last_template]
+        reordered += [c for c in candidates if c.template_id == last_template]
+        candidates = reordered
+
+    best: SentenceRecord | None = None
+    for cand in candidates:
+        generated = generate_from_plan(plan, template_id=cand.template_id)
+        if not generated:
+            continue
+        repaired = repair_sentence(generated) if GRAMMAR_FIX_OUTPUT else generated
+        safety = check_safety(
+            original,
+            repaired,
+            min_meaning=safety_min,
+            min_confidence=min_confidence,
+            use_minilm=use_minilm,
+        )
+        if not safety.ok:
+            stats.bump(safety.reasons[0] if safety.reasons else "safety")
+            continue
+        conf = min(cand.confidence, safety.confidence + 0.2)
+        best = SentenceRecord(
+            index=index,
+            original=original,
+            rewritten=repaired,
+            confidence=conf,
+            status="rewritten",
+            template_id=cand.template_id,
+            sentence_type=kind,
+            reasons=[],
+            block_index=block_index,
+        )
+        break
+
+    if best is None:
+        stats.reverted += 1
+        stats.bump("safety_fallback")
+        return (
+            SentenceRecord(
+                index=index,
+                original=original,
+                rewritten=original,
+                confidence=0.0,
+                status="reverted",
+                sentence_type=kind,
+                reasons=["safety_fallback"],
+                block_index=block_index,
+            ),
+            "",
+        )
+
+    stats.rewritten += 1
+    return best, best.template_id
+
+
+def _process_block(
+    block: DocumentBlock,
+    *,
+    sent_offset: int,
+    min_confidence: float,
+    safety_min: float,
+    use_minilm: bool,
+    last_template: str,
+    stats: EngineStats,
+) -> tuple[DocumentBlock, list[SentenceRecord], str]:
+    if not block.rewriteable or block.kind != "paragraph":
+        stats.passthrough += 1
+        rec = SentenceRecord(
+            index=sent_offset,
+            original=block.text,
+            rewritten=block.text,
+            confidence=1.0,
+            status="passthrough",
+            sentence_type=block.kind,
+            block_index=block.index,
+            reasons=[f"block:{block.kind}"],
+        )
+        return block, [rec], last_template
+
+    masked, prot = mask_protected_spans(block.text)
+    sentences = split_sentences(masked)
+    if not sentences:
+        return block, [], last_template
+
+    records: list[SentenceRecord] = []
+    out_sents: list[str] = []
+    tid = last_template
+    for i, sent in enumerate(sentences):
+        rec, tid = _process_sentence(
+            sent,
+            index=sent_offset + i,
+            block_index=block.index,
+            min_confidence=min_confidence,
+            safety_min=safety_min,
+            use_minilm=use_minilm,
+            last_template=tid,
+            stats=stats,
+        )
+        # Unmask protected spans in both sides of the record
+        rec.original = unmask_protected_spans(rec.original, prot)
+        rec.rewritten = unmask_protected_spans(rec.rewritten, prot)
+        records.append(rec)
+        out_sents.append(rec.rewritten)
+
+    new_text = unmask_protected_spans(join_sentences(out_sents), prot)
+    new_block = DocumentBlock(
+        text=new_text,
+        kind=block.kind,
+        rewriteable=block.rewriteable,
+        index=block.index,
+        meta=dict(block.meta),
+    )
+    return new_block, records, tid
+
+
+def rewrite_document(
+    source: str | bytes | None,
+    *,
+    batch_paras: int | None = None,
+    min_confidence: float | None = None,
+    safety_min: float | None = None,
+    use_minilm_safety: bool | None = None,
+) -> EngineResult:
+    """
+    Structural document rewrite: reorder safe sentences, skip the rest.
+
+    Processes large documents in paragraph batches. Tone/synonym replacement
+    is not used on this path.
+    """
+    started = time.perf_counter()
+    batch_paras = batch_paras if batch_paras is not None else ENGINE_BATCH_PARAS
+    min_confidence = (
+        min_confidence if min_confidence is not None else ENGINE_MIN_CONFIDENCE
+    )
+    safety_min = safety_min if safety_min is not None else ENGINE_SAFETY_MIN
+    use_minilm = (
+        use_minilm_safety
+        if use_minilm_safety is not None
+        else ENGINE_USE_MINILM_SAFETY
+    )
+
+    raw = ingest_text(source)
+    if not raw.strip():
+        raise ValueError("Paste some text first.")
+
+    normalized = normalize_text(raw)
+    if GRAMMAR_FIX_INPUT:
+        try:
+            from app.pipeline.grammar_fix import correct_text
+
+            cleaned = correct_text(normalized)
+            if cleaned and cleaned.strip():
+                normalized = normalize_text(cleaned)
+        except Exception:
+            logger.debug("input grammar fix skipped", exc_info=True)
+
+    blocks = detect_blocks(normalized)
+    stats = EngineStats(blocks=len(blocks))
+    all_records: list[SentenceRecord] = []
+    out_blocks: list[DocumentBlock] = []
+    sent_offset = 0
+    last_template = ""
+
+    for batch in iter_paragraph_batches(blocks, batch_paras=batch_paras):
+        stats.batches += 1
+        for block in batch:
+            new_block, records, last_template = _process_block(
+                block,
+                sent_offset=sent_offset,
+                min_confidence=min_confidence,
+                safety_min=safety_min,
+                use_minilm=use_minilm,
+                last_template=last_template,
+                stats=stats,
+            )
+            out_blocks.append(new_block)
+            all_records.extend(records)
+            # Advance offset by sentence count for rewriteable paras
+            if block.rewriteable and block.kind == "paragraph":
+                sent_offset += max(1, len(records))
+            else:
+                sent_offset += 1
+
+    stats.sentences = len(all_records)
+    stitched = stitch_blocks(out_blocks)
+
+    # Consistency: suppress streaks then rebuild from records if needed
+    stitched, all_records = apply_consistency(stitched, all_records)
+    streak_reverts = [r for r in all_records if "template_streak" in r.reasons]
+    if streak_reverts:
+        # Rebuild rewriteable paragraph texts from records
+        by_block: dict[int, list[SentenceRecord]] = {}
+        for rec in all_records:
+            by_block.setdefault(rec.block_index, []).append(rec)
+        rebuilt: list[DocumentBlock] = []
+        for block in out_blocks:
+            recs = by_block.get(block.index, [])
+            if block.rewriteable and block.kind == "paragraph" and recs:
+                text = join_sentences([r.rewritten for r in recs])
+                rebuilt.append(
+                    DocumentBlock(
+                        text=text,
+                        kind=block.kind,
+                        rewriteable=True,
+                        index=block.index,
+                    )
+                )
+            else:
+                rebuilt.append(block)
+        stitched = stitch_blocks(rebuilt)
+        stitched, all_records = apply_consistency(stitched, all_records)
+
+    stats.seconds = round(time.perf_counter() - started, 3)
+    mapping = [(r.original, r.rewritten) for r in all_records]
+    skipped = [r for r in all_records if r.status in {"skipped", "reverted"}]
+    in_words = word_count(normalized)
+    out_words = word_count(stitched)
+    changed = stitched.strip() != normalized.strip()
+    sim = _similarity(normalized, stitched)
+
+    notes = (
+        f"structural: rewritten={stats.rewritten} skipped={stats.skipped} "
+        f"reverted={stats.reverted} batches={stats.batches}"
+    )
+    logger.info(
+        "engine done words=%s→%s rewritten=%s skipped=%s reverted=%s batches=%s t=%.2fs",
+        in_words,
+        out_words,
+        stats.rewritten,
+        stats.skipped,
+        stats.reverted,
+        stats.batches,
+        stats.seconds,
+    )
+
+    return EngineResult(
+        text=stitched,
+        sentences=all_records,
+        skipped=skipped,
+        mapping=mapping,
+        stats=stats,
+        notes=notes,
+        engine="structural-reorder",
+        input_words=in_words,
+        output_words=out_words,
+        changed=changed,
+        similarity=round(sim, 4),
+    )

app/engine/parse/__init__.py

@@ -0,0 +1,217 @@
+"""Dependency parsing — spaCy-first slot extraction (no synonym tables)."""
+
+from __future__ import annotations
+
+import re
+
+from app.engine.classify import classify_sentence
+from app.engine.models import SentenceSlots
+from app.pipeline.nlp import get_nlp
+
+# Closed-class helpers only (spaCy NER/deps preferred for open-class time/place)
+_TIME_WORDS = frozenset(
+    """
+    yesterday today tomorrow earlier later recently now tonight
+    monday tuesday wednesday thursday friday saturday sunday
+    morning afternoon evening night
+    """.split()
+)
+_DISCOURSE_TIME = frozenset({"nowadays"})
+_DEGREE_ADVS = frozenset(
+    """
+    more most less least much many very really quite just also still even
+    only rather pretty fairly so too enough almost nearly already ever never
+    not n't however nevertheless furthermore moreover
+    """.split()
+)
+_PLACE_PREPS = frozenset("to at in on into onto from toward towards".split())
+
+
+def _span_text(tokens: list) -> str:
+    if not tokens:
+        return ""
+    return "".join(t.text_with_ws for t in tokens).strip()
+
+
+def _subtree_tokens(token) -> list:
+    return sorted(token.subtree, key=lambda t: t.i)
+
+
+def _is_manner_adv(word: str) -> bool:
+    low = (word or "").lower().strip()
+    if not low or low in _DEGREE_ADVS or low in _DISCOURSE_TIME:
+        return False
+    return low.endswith("ly") and len(low) >= 4
+
+
+def extract_slots(text: str) -> SentenceSlots:
+    """Parse subject/verb/object/time/place/manner/negation/entities."""
+    raw = (text or "").strip()
+    slots = SentenceSlots(text=raw, sentence_type=classify_sentence(raw))
+    if slots.sentence_type not in {"simple_declarative", "compound"}:
+        slots.reasons.append(f"skip:{slots.sentence_type}")
+        return slots
+
+    nlp = get_nlp()
+    if nlp is None:
+        return _extract_slots_regex(raw)
+
+    doc = nlp(raw)
+    slots.entities = [ent.text for ent in doc.ents if ent.label_ in {
+        "PERSON", "ORG", "GPE", "LOC", "DATE", "TIME", "MONEY", "PERCENT", "CARDINAL",
+    }]
+
+    root = next((t for t in doc if t.dep_ == "ROOT" and t.pos_ in {"VERB", "AUX"}), None)
+    if root is None:
+        root = next((t for t in doc if t.pos_ == "VERB"), None)
+    if root is None:
+        slots.reasons.append("no_verb")
+        return slots
+
+    subj_toks: list = []
+    for child in root.children:
+        if child.dep_ in {"nsubj", "nsubjpass"}:
+            subj_toks = _subtree_tokens(child)
+            break
+    slots.subject = _span_text(subj_toks)
+
+    neg_parts: list[str] = []
+    for t in doc:
+        if t.dep_ == "neg" and (t.head == root or t.head.head == root):
+            neg_parts.append(t.text)
+    slots.negation = " ".join(neg_parts)
+
+    verb_toks = [t for t in root.lefts if t.dep_ in {"aux", "auxpass", "neg"}] + [root]
+    slots.verb = root.text
+    slots.verb_phrase = _span_text(sorted(verb_toks, key=lambda t: t.i))
+
+    obj_toks: list = []
+    for child in root.children:
+        if child.dep_ in {"dobj", "obj", "attr"}:
+            obj_toks = _subtree_tokens(child)
+            break
+        if child.dep_ in {"ccomp", "xcomp"} and child.pos_ != "VERB":
+            obj_toks = _subtree_tokens(child)
+            break
+    slots.object = _span_text(obj_toks)
+
+    place_parts: list[str] = []
+    for child in root.children:
+        if child.dep_ == "prep" and child.text.lower() in _PLACE_PREPS:
+            # Prefer place over time-marked preps
+            span = _span_text(_subtree_tokens(child))
+            if not any(w.lower() in _TIME_WORDS for w in span.split()):
+                place_parts.append(span)
+    if not place_parts:
+        for t in doc:
+            if t.dep_ == "prep" and t.head == root and t.text.lower() in _PLACE_PREPS:
+                place_parts.append(_span_text(_subtree_tokens(t)))
+    slots.place = " ".join(p for p in place_parts if p).strip()
+
+    time_parts: list[str] = []
+    manner_parts: list[str] = []
+    for ent in doc.ents:
+        if ent.label_ in {"DATE", "TIME"}:
+            time_parts.append(ent.text)
+    for t in doc:
+        low = t.text.lower()
+        if t.dep_ in {"advmod", "npadvmod"} and (t.head == root or t.head.head == root):
+            if low in _TIME_WORDS or t.ent_type_ in {"DATE", "TIME"}:
+                if low not in _DISCOURSE_TIME:
+                    time_parts.append(_span_text(_subtree_tokens(t)))
+            elif _is_manner_adv(low):
+                manner_parts.append(t.text)
+        if t.dep_ == "npadvmod" and low in _TIME_WORDS and low not in _DISCOURSE_TIME:
+            time_parts.append(t.text)
+
+    if not time_parts:
+        for t in doc:
+            low = t.text.lower()
+            if low in _TIME_WORDS and low not in _DISCOURSE_TIME:
+                time_parts.append(t.text)
+    if not manner_parts:
+        for t in doc:
+            if _is_manner_adv(t.text.lower()) and t.pos_ == "ADV":
+                manner_parts.append(t.text)
+
+    slots.time = " ".join(dict.fromkeys(time_parts)).strip()
+    slots.manner = " ".join(dict.fromkeys(manner_parts)).strip()
+
+    for piece in (slots.place, slots.time, slots.manner):
+        if piece and slots.object.endswith(piece):
+            slots.object = slots.object[: -len(piece)].strip(" ,")
+
+    score = 0.2
+    if slots.subject:
+        score += 0.25
+    if slots.verb_phrase:
+        score += 0.2
+    if slots.place or slots.object:
+        score += 0.15
+    if slots.time:
+        score += 0.1
+    if slots.manner:
+        score += 0.1
+    slots.confidence = min(1.0, score)
+    if slots.confidence < 0.45 or not slots.subject or not slots.verb_phrase:
+        slots.reasons.append("low_confidence")
+    return slots
+
+
+def _extract_slots_regex(text: str) -> SentenceSlots:
+    """Regex fallback when spaCy is unavailable."""
+    slots = SentenceSlots(text=text, sentence_type=classify_sentence(text))
+    if slots.sentence_type != "simple_declarative":
+        return slots
+    core = text
+    m_end = re.search(r"[.!?]+$", text)
+    if m_end:
+        core = text[: m_end.start()]
+
+    m = re.match(
+        r"^(?P<subj>[A-Za-z][\w'-]*)\s+"
+        r"(?P<verb>\w+)\s+"
+        r"(?:to\s+(?P<place>.+?)\s+)?"
+        r"(?P<time>yesterday|today|tomorrow|earlier|later|recently)?\s*"
+        r"(?P<manner>\w+ly)?$",
+        core,
+        flags=re.I,
+    )
+    if m:
+        slots.subject = m.group("subj")
+        slots.verb = m.group("verb")
+        slots.verb_phrase = m.group("verb")
+        if m.group("place"):
+            slots.place = "to " + m.group("place").strip()
+        slots.time = (m.group("time") or "").strip()
+        slots.manner = (m.group("manner") or "").strip()
+        slots.confidence = 0.7
+        return slots
+
+    m2 = re.match(r"^(?P<subj>[A-Za-z][\w'-]*)\s+(?P<rest>.+)$", core, flags=re.I)
+    if not m2:
+        slots.reasons.append("regex_no_match")
+        return slots
+    slots.subject = m2.group("subj")
+    rest = m2.group("rest")
+    mm = re.search(r"\b(\w+ly)$", rest, flags=re.I)
+    if mm:
+        slots.manner = mm.group(1)
+        rest = rest[: mm.start()].strip()
+    mt = re.search(
+        r"\b(yesterday|today|tomorrow|earlier|later|recently)$",
+        rest,
+        flags=re.I,
+    )
+    if mt:
+        slots.time = mt.group(1)
+        rest = rest[: mt.start()].strip()
+    mv = re.match(r"^(\w+)(?:\s+to\s+(.+))?$", rest, flags=re.I)
+    if mv:
+        slots.verb_phrase = mv.group(1)
+        slots.verb = mv.group(1)
+        slots.place = f"to {mv.group(2)}" if mv.group(2) else ""
+    slots.confidence = 0.55 if slots.subject and slots.verb_phrase else 0.2
+    if slots.confidence < 0.45:
+        slots.reasons.append("low_confidence")
+    return slots

app/engine/plan/__init__.py

@@ -0,0 +1,97 @@
+"""Rewrite planning — decide safety and ranked template candidates."""
+
+from __future__ import annotations
+
+from app.engine.classify import classify_sentence, is_rewriteable_type
+from app.engine.models import RewritePlan, SentenceSlots, TemplateCandidate
+from app.engine.parse import extract_slots
+from app.engine.templates import rank_templates
+
+
+def build_plan(text: str, *, min_confidence: float = 0.55) -> RewritePlan:
+    """Create a rewrite plan without generating text yet."""
+    raw = (text or "").strip()
+    if not raw:
+        return RewritePlan(safe=False, skip_reason="empty")
+
+    kind = classify_sentence(raw)
+    if kind == "because_clause":
+        return RewritePlan(
+            safe=True,
+            template_id="because_front",
+            candidates=[TemplateCandidate("because_front", 0.85)],
+            confidence=0.85,
+            slots=SentenceSlots(text=raw, sentence_type=kind, confidence=0.85),
+            movable=["because_clause"],
+            fixed_spans=[],
+        )
+
+    if not is_rewriteable_type(kind):
+        return RewritePlan(
+            safe=False,
+            skip_reason=kind,
+            confidence=0.0,
+            slots=SentenceSlots(text=raw, sentence_type=kind),
+        )
+
+    # Discourse front as dedicated candidate when applicable
+    low = raw.lower()
+    if "nowadays" in low and not low.startswith("nowadays"):
+        return RewritePlan(
+            safe=True,
+            template_id="discourse_front",
+            candidates=[TemplateCandidate("discourse_front", 0.7)],
+            confidence=0.7,
+            slots=SentenceSlots(text=raw, sentence_type=kind, confidence=0.7),
+            movable=["discourse"],
+        )
+
+    slots = extract_slots(raw)
+    if "low_confidence" in slots.reasons or slots.confidence < min_confidence:
+        return RewritePlan(
+            safe=False,
+            skip_reason="low_confidence",
+            confidence=slots.confidence,
+            slots=slots,
+        )
+    if any(r.startswith("skip:") for r in slots.reasons):
+        return RewritePlan(
+            safe=False,
+            skip_reason=slots.reasons[0],
+            confidence=slots.confidence,
+            slots=slots,
+        )
+
+    candidates = rank_templates(slots)
+    if not candidates:
+        return RewritePlan(
+            safe=False,
+            skip_reason="no_template",
+            confidence=slots.confidence,
+            slots=slots,
+        )
+
+    # Filter by min confidence
+    candidates = [c for c in candidates if c.confidence >= min_confidence]
+    if not candidates:
+        return RewritePlan(
+            safe=False,
+            skip_reason="low_template_confidence",
+            confidence=slots.confidence,
+            slots=slots,
+        )
+
+    fixed = list(slots.entities)
+    if slots.negation:
+        fixed.append(slots.negation)
+    movable = [x for x in ("time", "manner", "place") if getattr(slots, x, "")]
+
+    return RewritePlan(
+        safe=True,
+        slots=slots,
+        template_id=candidates[0].template_id,
+        candidates=candidates,
+        fixed_spans=fixed,
+        movable=movable,
+        confidence=candidates[0].confidence,
+    )

app/engine/rewrite/__init__.py

@@ -0,0 +1,85 @@
+"""Sentence rewrite generation from plans/templates."""
+
+from __future__ import annotations
+
+import re
+
+from app.engine.models import RewritePlan
+from app.engine.templates import fill_template, try_because_front, try_discourse_front
+
+
+def _content_tokens(text: str) -> list[str]:
+    return [
+        w
+        for w in re.findall(r"[a-zA-Z']+", (text or "").lower())
+        if len(w) >= 3
+    ]
+
+
+def reorder_quality_ok(source: str, candidate: str) -> bool:
+    """Reject junk reorders that drop clauses or shuffle degree adverbs."""
+    src = (source or "").strip()
+    cand = (candidate or "").strip()
+    if not src or not cand:
+        return False
+    if cand.lower().rstrip(".!?") == src.lower().rstrip(".!?"):
+        return False
+    if re.search(
+        r"\b(more|most|less|least|very|really|quite)\s+"
+        r"(makes?|make|is|are|was|were|has|have|had|does|do|did)\b",
+        cand,
+        flags=re.I,
+    ):
+        return False
+    src_toks = _content_tokens(src)
+    cand_toks = _content_tokens(cand)
+    if not src_toks:
+        return False
+    keep = len(set(src_toks) & set(cand_toks)) / max(1, len(set(src_toks)))
+    if keep < 0.85:
+        return False
+    if len(cand_toks) < int(len(src_toks) * 0.75) or len(cand_toks) > int(
+        len(src_toks) * 1.35
+    ):
+        return False
+    return True
+
+
+def generate_from_plan(
+    plan: RewritePlan,
+    *,
+    template_id: str | None = None,
+) -> str | None:
+    """Generate a rewritten sentence from a plan (slot reorder only)."""
+    if not plan.safe:
+        return None
+    tid = template_id or plan.template_id
+    raw = plan.slots.text if plan.slots else ""
+
+    if tid == "because_front":
+        return try_because_front(raw)
+    if tid == "discourse_front":
+        return try_discourse_front(raw)
+
+    if plan.slots is None:
+        return None
+    filled = fill_template(tid, plan.slots)
+    if not filled:
+        return None
+    if filled.strip().lower().rstrip(".!?") == raw.lower().rstrip(".!?"):
+        return None
+    if not reorder_quality_ok(raw, filled):
+        return None
+    return filled.strip()
+
+
+def generate_candidates(plan: RewritePlan) -> list[tuple[str, str, float]]:
+    """Try each ranked template; return (template_id, text, confidence)."""
+    results: list[tuple[str, str, float]] = []
+    if not plan.safe:
+        return results
+    for cand in plan.candidates:
+        text = generate_from_plan(plan, template_id=cand.template_id)
+        if text:
+            results.append((cand.template_id, text, cand.confidence))
+    return results

app/engine/safety/__init__.py

@@ -0,0 +1,145 @@
+"""Semantic safety checks and fallback decisions."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from app.pipeline.candidate_validator import validate_candidate
+from app.pipeline.meaning_safety import polarity_safe
+
+_URL = re.compile(r"https?://[^\s<>\"']+|www\.[^\s<>\"']+", re.I)
+_EMAIL = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b")
+_NUMBER = re.compile(r"\b\d[\d,]*(?:\.\d+)?%?\b")
+
+
+@dataclass
+class SafetyResult:
+    ok: bool
+    confidence: float
+    reasons: list[str]
+    surface_sim: float = 0.0
+    meaning: float = 0.0
+
+
+def _entity_tokens(text: str) -> set[str]:
+    """Heuristic proper-noun / protected token set."""
+    toks = set()
+    for m in re.finditer(r"\b[A-Z][a-zA-Z0-9'-]+\b", text or ""):
+        # Skip sentence-initial common words later; keep all for subset check
+        toks.add(m.group(0))
+    for m in _URL.finditer(text or ""):
+        toks.add(m.group(0))
+    for m in _EMAIL.finditer(text or ""):
+        toks.add(m.group(0))
+    return toks
+
+
+def _numbers(text: str) -> set[str]:
+    return {m.group(0).replace(",", "") for m in _NUMBER.finditer(text or "")}
+
+
+def _tense_aux_ok(original: str, candidate: str) -> bool:
+    """Reject if auxiliary/tense markers disappear."""
+    aux = {"did", "does", "do", "was", "were", "is", "are", "am", "has", "have", "had", "will", "would", "could", "should"}
+    o = set(re.findall(r"[a-zA-Z']+", (original or "").lower()))
+    c = set(re.findall(r"[a-zA-Z']+", (candidate or "").lower()))
+    o_aux = o & aux
+    if not o_aux:
+        return True
+    return o_aux.issubset(c)
+
+
+def check_safety(
+    original: str,
+    candidate: str,
+    *,
+    min_meaning: float = 0.80,
+    min_confidence: float = 0.55,
+    use_minilm: bool = False,
+) -> SafetyResult:
+    """Lightweight similarity/safety gate between original and rewrite."""
+    reasons: list[str] = []
+    o = (original or "").strip()
+    c = (candidate or "").strip()
+    if not o or not c:
+        return SafetyResult(False, 0.0, ["empty"])
+
+    if not polarity_safe(o, c):
+        reasons.append("negation")
+
+    o_ents = _entity_tokens(o)
+    # Drop first token of original if capitalized (sentence start)
+    first = (o.split() or [""])[0].strip(".,;:!?\"'")
+    o_ents.discard(first)
+    for ent in o_ents:
+        if ent not in c and ent.lower() not in c.lower():
+            reasons.append(f"entity:{ent}")
+            break
+
+    o_nums, c_nums = _numbers(o), _numbers(c)
+    if o_nums and not o_nums.issubset(c_nums):
+        reasons.append("numbers")
+
+    if not _tense_aux_ok(o, c):
+        reasons.append("tense")
+
+    # Structural reorder may be near-copy in surface ratio; relax max_surface
+    vr = validate_candidate(
+        o,
+        c,
+        min_meaning=min_meaning if use_minilm else 0.0,
+        max_surface=0.995,
+        min_surface=0.20,
+    )
+    # Filter validator reasons that fight structural reorder
+    ignore = {"too_similar", "identical"}
+    for r in vr.reasons:
+        if r in ignore:
+            continue
+        if r.startswith("meaning:") and not use_minilm:
+            continue
+        if r not in reasons:
+            reasons.append(r)
+
+    # Optional MiniLM meaning score when enabled
+    meaning = vr.meaning
+    if use_minilm:
+        try:
+            from app.pipeline.minilm import score_candidate
+
+            scored = score_candidate(o, c)
+            if scored is not None:
+                meaning = float(scored)
+                if meaning < min_meaning:
+                    reasons.append(f"meaning:{meaning:.2f}")
+        except Exception:
+            pass
+
+    confidence = max(0.0, min(1.0, (meaning + (1.0 - abs(vr.surface_sim - 0.7))) / 2))
+    if reasons:
+        confidence = min(confidence, 0.4)
+
+    ok = not reasons and confidence >= min_confidence * 0.5
+    # If only soft issues, still allow when polarity+entities ok
+    hard = {
+        r
+        for r in reasons
+        if r in {"negation", "numbers", "tense", "polarity", "entity_inject", "invention", "broken"}
+        or r.startswith("entity:")
+        or r.startswith("meaning:")
+    }
+    if hard:
+        ok = False
+    elif reasons and vr.surface_sim >= 0.35:
+        # Soft validator noise on reorders — accept if content preserved
+        ok = True
+        confidence = max(confidence, 0.6)
+
+    return SafetyResult(
+        ok=ok,
+        confidence=confidence,
+        reasons=reasons,
+        surface_sim=vr.surface_sim,
+        meaning=float(meaning or 0.0),
+    )

app/engine/segment/__init__.py

@@ -0,0 +1,58 @@
+"""Paragraph / sentence segmentation with batch streaming."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+from app.engine.models import DocumentBlock
+from app.pipeline.nlp import get_nlp
+from app.pipeline.normalize import split_sentences_regex
+
+
+def split_sentences(text: str) -> list[str]:
+    """Split a paragraph into sentences (spaCy preferred)."""
+    raw = (text or "").strip()
+    if not raw:
+        return []
+    nlp = get_nlp()
+    if nlp is not None:
+        try:
+            doc = nlp(raw)
+            sents = [s.text.strip() for s in doc.sents if s.text.strip()]
+            if sents:
+                return sents
+        except Exception:
+            pass
+    return split_sentences_regex(raw)
+
+
+def iter_paragraph_batches(
+    blocks: list[DocumentBlock],
+    *,
+    batch_paras: int = 20,
+) -> Iterator[list[DocumentBlock]]:
+    """Yield batches of rewriteable paragraph blocks interleaved with passthrough.
+
+    Each yielded list preserves document order for that slice of blocks.
+    Non-rewriteable blocks are included so stitch can keep structure.
+    Batching is driven by count of rewriteable paragraphs.
+    """
+    batch_paras = max(1, min(int(batch_paras), 100))
+    batch: list[DocumentBlock] = []
+    rewriteable_count = 0
+
+    for block in blocks:
+        batch.append(block)
+        if block.rewriteable and block.kind == "paragraph":
+            rewriteable_count += 1
+        if rewriteable_count >= batch_paras:
+            yield batch
+            batch = []
+            rewriteable_count = 0
+
+    if batch:
+        yield batch
+
+
+def word_count(text: str) -> int:
+    return len((text or "").split()) if (text or "").strip() else 0

app/engine/stitch/__init__.py

@@ -0,0 +1,35 @@
+"""Rebuild document text from processed blocks."""
+
+from __future__ import annotations
+
+from app.engine.models import DocumentBlock
+
+
+def stitch_blocks(blocks: list[DocumentBlock]) -> str:
+    """Join blocks preserving blank lines and non-rewriteable sections."""
+    if not blocks:
+        return ""
+    parts: list[str] = []
+    for i, block in enumerate(blocks):
+        if block.kind == "blank":
+            # Represented as paragraph separator; avoid stacking too many
+            if parts and not parts[-1].endswith("\n\n"):
+                if parts[-1] and not parts[-1].endswith("\n"):
+                    parts.append("\n\n")
+                elif parts[-1].endswith("\n") and not parts[-1].endswith("\n\n"):
+                    parts.append("\n")
+            continue
+        text = block.text or ""
+        if i > 0 and parts and not parts[-1].endswith("\n"):
+            # Separate consecutive non-blank blocks with blank line if both are paras/headings
+            prev = blocks[i - 1]
+            if prev.kind != "blank":
+                parts.append("\n\n")
+        parts.append(text)
+    out = "".join(parts)
+    out = out.replace("\n\n\n\n", "\n\n").strip()
+    return out
+
+
+def join_sentences(sentences: list[str]) -> str:
+    return " ".join(s.strip() for s in sentences if s and s.strip())

app/engine/templates/__init__.py

@@ -0,0 +1,182 @@
+"""Deterministic template families for structural reorder."""
+
+from __future__ import annotations
+
+import re
+
+from app.engine.models import SentenceSlots, TemplateCandidate
+
+
+def rank_templates(slots: SentenceSlots) -> list[TemplateCandidate]:
+    """Return confidence-ranked template candidates (never random)."""
+    if not slots.subject or not slots.verb_phrase:
+        return []
+    if slots.confidence < 0.45 or "low_confidence" in slots.reasons:
+        return []
+
+    out: list[TemplateCandidate] = []
+
+    if slots.time and slots.manner and (slots.place or slots.object):
+        out.append(TemplateCandidate("time_subj_manner_verb_place", 0.9))
+    if slots.time and slots.place:
+        out.append(TemplateCandidate("time_subj_verb_place", 0.8))
+    if slots.time and slots.object:
+        out.append(TemplateCandidate("time_subj_verb_object", 0.75))
+    if slots.time:
+        out.append(TemplateCandidate("time_subj_verb_object_place", 0.72))
+        out.append(TemplateCandidate("subj_verb_object_time", 0.68))
+        out.append(TemplateCandidate("time_front", 0.65))
+    if slots.manner and slots.place:
+        out.append(TemplateCandidate("subj_manner_verb_place", 0.7))
+    if slots.manner:
+        out.append(TemplateCandidate("adv_subj_verb_object", 0.62))
+        out.append(TemplateCandidate("subj_manner_verb_rest", 0.6))
+    if slots.place:
+        out.append(TemplateCandidate("subj_verb_place", 0.55))
+    if slots.object:
+        out.append(TemplateCandidate("subj_verb_object", 0.5))
+
+    # Dedupe by template_id keeping highest confidence order
+    seen: set[str] = set()
+    ranked: list[TemplateCandidate] = []
+    for c in sorted(out, key=lambda x: -x.confidence):
+        if c.template_id in seen:
+            continue
+        seen.add(c.template_id)
+        ranked.append(c)
+    return ranked
+
+
+def _cap(text: str) -> str:
+    t = (text or "").strip()
+    if not t:
+        return t
+    return t[0].upper() + t[1:]
+
+
+def _join_slots(*parts: str) -> str:
+    bits = [p.strip() for p in parts if p and p.strip()]
+    s = " ".join(bits)
+    s = re.sub(r"\s+", " ", s).strip()
+    s = re.sub(r"\s+([,.;:!?])", r"\1", s)
+    return s
+
+
+def _terminal(text: str) -> str:
+    if text.rstrip().endswith(("!", "?")):
+        return text.rstrip()[-1]
+    return "."
+
+
+def fill_template(template_id: str, slots: SentenceSlots) -> str | None:
+    """Build sentence from slots using the selected template (reorder only)."""
+    s = slots
+    end = _terminal(s.text)
+    tid = template_id
+
+    if tid == "time_subj_manner_verb_place":
+        place_or_obj = s.place or s.object
+        body = _join_slots(_cap(s.time) + ",", s.subject, s.manner, s.verb_phrase, place_or_obj)
+        return body + end
+
+    if tid == "time_subj_verb_place":
+        body = _join_slots(_cap(s.time) + ",", s.subject, s.verb_phrase, s.place)
+        return body + end
+
+    if tid == "time_subj_verb_object":
+        body = _join_slots(_cap(s.time) + ",", s.subject, s.verb_phrase, s.object)
+        return body + end
+
+    if tid == "time_subj_verb_object_place":
+        body = _join_slots(
+            _cap(s.time) + ",", s.subject, s.verb_phrase, s.object, s.place, s.manner
+        )
+        return body + end
+
+    if tid == "subj_verb_object_time":
+        body = _join_slots(_cap(s.subject), s.verb_phrase, s.object, s.place, s.manner, s.time)
+        return body + end
+
+    if tid == "adv_subj_verb_object":
+        body = _join_slots(_cap(s.manner) + ",", s.subject, s.verb_phrase, s.object, s.place, s.time)
+        return body + end
+
+    if tid == "subj_manner_verb_place":
+        body = _join_slots(_cap(s.subject), s.manner, s.verb_phrase, s.place)
+        return body + end
+
+    if tid == "time_front":
+        rest = s.text
+        if s.time:
+            rest = re.sub(rf"\b{re.escape(s.time)}\b", "", rest, count=1, flags=re.I)
+            rest = re.sub(r"\s+", " ", rest).strip(" ,.")
+        if not rest:
+            return None
+        body = _join_slots(_cap(s.time) + ",", rest[0].lower() + rest[1:])
+        return body.rstrip(".!?") + end
+
+    if tid == "subj_manner_verb_rest":
+        rest_bits = [s.verb_phrase, s.object, s.place, s.time]
+        body = _join_slots(_cap(s.subject), s.manner, *rest_bits)
+        return body + end
+
+    if tid == "subj_verb_place":
+        body = _join_slots(_cap(s.subject), s.verb_phrase, s.place, s.time, s.manner)
+        return body + end
+
+    if tid == "subj_verb_object":
+        body = _join_slots(_cap(s.subject), s.verb_phrase, s.object, s.place, s.time, s.manner)
+        return body + end
+
+    return None
+
+
+def try_because_front(text: str) -> str | None:
+    """Main … because Sub → Because Sub, main …"""
+    raw = (text or "").strip()
+    if not raw:
+        return None
+    end = "."
+    m_end = re.search(r"[.!?]+$", raw)
+    if m_end:
+        end = m_end.group(0)
+        raw = raw[: m_end.start()].strip()
+    m = re.match(r"^(?P<main>.+?)\s+because\s+(?P<sub>.+)$", raw, flags=re.I)
+    if not m:
+        return None
+    main = m.group("main").strip(" ,")
+    sub = m.group("sub").strip(" ,")
+    if len(main.split()) < 3 or len(sub.split()) < 2:
+        return None
+    if re.search(r"\bbecause\b", main, flags=re.I) or re.search(
+        r"\bbecause\b", sub, flags=re.I
+    ):
+        return None
+    body = (
+        f"Because {sub[0].lower() + sub[1:]}, "
+        f"{main[0].lower() + main[1:]}"
+    )
+    return re.sub(r"\s+", " ", body).strip() + end
+
+
+def try_discourse_front(text: str) -> str | None:
+    """Move mid-sentence 'nowadays' to the front."""
+    raw = (text or "").strip()
+    if not raw or re.match(r"^nowadays\b", raw, flags=re.I):
+        return None
+    if re.search(r"\b(because|although|while|unless)\b", raw, flags=re.I):
+        return None
+    end = "."
+    m_end = re.search(r"[.!?]+$", raw)
+    core = raw
+    if m_end:
+        end = m_end.group(0)
+        core = raw[: m_end.start()].strip()
+    m = re.search(r"\bnowadays\b", core, flags=re.I)
+    if not m:
+        return None
+    rest = (core[: m.start()] + core[m.end() :]).strip(" ,")
+    if len(rest.split()) < 4:
+        return None
+    out = f"Nowadays, {rest[0].lower() + rest[1:]}{end}"
+    return re.sub(r"\s+", " ", out).strip()

app/main.py

@@ -29,8 +29,6 @@ from app.bootstrap import ensure_resources
 from app.config import (
     APP_TITLE,
     AUTH_ENABLED,
-    GENERATIVE_BACKEND,
-    GENERATIVE_MODEL,
     GRAMMAR_MAX_CHARS,
     LANGUAGE_TOOL_LANGUAGE,
     LANGUAGE_TOOL_URL,
@@ -38,7 +36,6 @@ from app.config import (
     ML_POLISH_AVAILABLE_DEFAULT,
     PIPELINE_MODE,
 )
-from app.pipeline.generative import backend_kind, generative_package_present
 from app.pipeline.grammar import (
     check_grammar,
     languagetool_reachable,
@@ -98,18 +95,14 @@ def _client_ip(request: Request) -> str:
 
 @app.get("/health")
 def health():
-    engines = ["rules", "wordnet", "mechanics", "grammar"]
+    engines = ["structural-reorder", "grammar"]
     engines.insert(0, "spacy" if spacy_available() else "regex-fallback")
     lt_ok = languagetool_reachable()
     if lt_ok:
         engines.append("languagetool")
     ml_pkg = minilm_package_present()
-    gen_pkg = generative_package_present()
     if ml_pkg:
-        engines.append("minilm")
-    if gen_pkg:
-        engines.append("generative")
-        engines.append(GENERATIVE_BACKEND)
+        engines.append("minilm-safety")
     return {
         "status": "ok",
         "app": APP_TITLE,
@@ -122,24 +115,41 @@ def health():
             "reachable": lt_ok,
             "default_language": LANGUAGE_TOOL_LANGUAGE or "en-US",
         },
+        "rewrite_engine": {
+            "mode": "structural",
+            "pipeline_mode": PIPELINE_MODE,
+            "pipeline": [
+                "ingest",
+                "normalize",
+                "segment (batched)",
+                "classify",
+                "parse (spaCy)",
+                "plan",
+                "templates",
+                "rewrite (reorder)",
+                "grammar",
+                "safety",
+                "stitch",
+                "consistency",
+            ],
+            "minilm_safety": ml_pkg,
+        },
         "ml_polish": {
             "enabled": bool(ML_POLISH_AVAILABLE_DEFAULT),
-            "available": bool(ml_pkg or gen_pkg),
+            "available": bool(ml_pkg),
             "minilm": ml_pkg,
-            "generative": gen_pkg,
+            "generative": False,
             "pipeline_mode": PIPELINE_MODE,
             "backend": {
                 "minilm": "fastembed|sentence-transformers" if ml_pkg else None,
-                "generative": GENERATIVE_MODEL if gen_pkg else None,
-                "generative_backend": GENERATIVE_BACKEND if gen_pkg else None,
-                "kind": backend_kind() if gen_pkg else None,
+                "generative": None,
+                "generative_backend": None,
+                "kind": None,
             },
             "pipeline": [
+                "structural-reorder",
                 "grammar",
-                "lm-unit rewrite (seq2seq|causal)",
-                "validate/minilm",
-                "rules-light",
-                "lexicon-fallback-if-no-lm",
+                "safety/minilm-optional",
             ],
             "plans": "all",
         },
@@ -203,6 +213,9 @@ def api_rewrite(
 
     return {
         "rewrite": result.text,
+        "sentences": result.sentences,
+        "mapping": result.mapping,
+        "stats": result.stats,
         "meta": {
             "engine": result.engine,
             "input_words": result.input_words,
@@ -215,8 +228,8 @@ def api_rewrite(
             "notes": result.notes,
             "ml_polish": use_ml,
             "pipeline_mode": result.pipeline_mode,
-            "generative_backend": GENERATIVE_BACKEND if use_ml else None,
-            "generative_model": GENERATIVE_MODEL if use_ml else None,
+            "generative_backend": None,
+            "generative_model": None,
             "hybrid": (
                 {
                     "units": result.hybrid.units,

app/pipeline/generative.py

@@ -1,548 +1,36 @@
-"""Generative rewrite: seq2seq (FLAN-T5) or causal chat (SmolLM2).
+"""Legacy generative LM rewrite — NOT used by the structural engine.
 
-Pipeline when models load:
-  1) Draft several paraphrases per unit
-  2) Validate polarity / facts / meaning (reject near-copies)
-  3) Rank survivors for fidelity + useful difference
-
-Lazy-loaded; missing torch/transformers → callers fall back to classical rules.
+Functions remain as stubs so optional imports and old scripts do not crash.
 """
 
 from __future__ import annotations
 
-import logging
-import re
-import threading
-from difflib import SequenceMatcher
 from typing import Any
 
-from app.config import (
-    GENERATIVE_BACKEND,
-    GENERATIVE_MAX_PARAGRAPHS,
-    GENERATIVE_MODEL,
-    GENERATIVE_POLISH_ENABLED,
-    ML_POLISH_AVAILABLE_DEFAULT,
-)
-from app.pipeline.candidate_ranker import NEAR_COPY_SURFACE, pick_best_candidate
-from app.pipeline.normalize import split_paragraphs
-from app.pipeline.tones import normalize_tone
-
-logger = logging.getLogger("plainrewrite.generative")
-
-_lock = threading.Lock()
-_tokenizer: Any = None
-_model: Any = None
-_failed = False
-_backend: str | None = None
-_backend_kind: str | None = None  # seq2seq | causal
-
-_SYSTEM_REWRITE = (
-    "You rewrite ONE English sentence with clearly DIFFERENT WORDS. "
-    "Output exactly one sentence — no second sentence, no preamble. "
-    "Keep EVERY claim and the same polarity. "
-    "Do not flip negatives or antonyms. "
-    "Preserve names, numbers, dates, quotations, and fixed phrases "
-    "(e.g. keep 'fast food' as 'fast food', not 'food' alone). "
-    "Do not add reasons, examples, education, awareness, routines, "
-    "workouts, diets, or new facts. "
-    "Do not summarize or drop content. "
-    "Do NOT rearrange the same words. Do NOT copy almost verbatim. "
-    "Replace verbs, adverbs, and phrases with natural synonyms. "
-    "Return only the rewritten sentence."
-)
-
-_SHARED_RULES = (
-    "Output exactly one sentence if the input is one sentence.\n"
-    "Keep ALL claims — do not drop, merge away, or invent ideas.\n"
-    "Do not add reasons, education, awareness, routines, workouts, diets, "
-    "or extra clauses.\n"
-    "Keep multiword terms intact (e.g. fast food, junk food, physical activity).\n"
-    "MUST use different wording — never rearrange the same words only.\n"
-    "Keep polarity. Preserve names, numbers, dates, and quotations. Correct grammar.\n"
-    "Return only the rewritten text."
-)
-
-_TONE_PROMPTS: dict[str, str] = {
-    "Casual": (
-        "Paraphrase this text in clear casual English.\n"
-        f"{_SHARED_RULES}\n\n"
-        "Text:\n{text}\n\n"
-        "Paraphrase:"
-    ),
-    "Formal": (
-        "Paraphrase this text in clear formal professional English.\n"
-        f"{_SHARED_RULES}\n\n"
-        "Text:\n{text}\n\n"
-        "Paraphrase:"
-    ),
-    "Academic": (
-        "Paraphrase this text in clear academic English.\n"
-        f"{_SHARED_RULES}\n\n"
-        "Text:\n{text}\n\n"
-        "Paraphrase:"
-    ),
-    "Neutral": (
-        "Paraphrase this text in clear natural English using different words.\n"
-        f"{_SHARED_RULES}\n"
-        "Keep polarity (e.g. unhealthy stays unhealthy; "
-        "don't realize stays a lack of awareness — without adding 'education').\n"
-        "Do not return a grammar-only edit or a shuffle of the same words.\n\n"
-        "Text:\n{text}\n\n"
-        "Paraphrase:"
-    ),
-}
-
-_RETRY_PROMPT = (
-    "Rewrite this as ONE sentence using DIFFERENT WORDS (not a word shuffle) "
-    "but the exact same meaning and polarity. Do not add or remove facts. "
-    "Keep fixed phrases like 'fast food'. "
-    "Do not invent workouts, diets, or routines. Do not add a second sentence. "
-    "Do not return a near-copy, grammar-only edit, or rearranged same words. "
-    "Tone: {tone}.\n\n"
-    "Text:\n{text}\n\n"
-    "Rewrite:"
-)
-
-_CAUSAL_USER = (
-    "Tone: {tone}.\n"
-    "Rewrite the following as exactly ONE sentence. "
-    "Keep every claim and polarity. Keep fixed phrases (e.g. fast food). "
-    "Do not drop, invent, or add education/routines/workouts.\n\n"
-    "{text}"
-)
-
-_CAUSAL_RETRY_USER = (
-    "Tone: {tone}.\n"
-    "Try again: exactly ONE sentence, different wording, same meaning and polarity. "
-    "No extra claims. No second sentence.\n\n"
-    "{text}"
-)
-
 
 def generative_package_present() -> bool:
-    if not ML_POLISH_AVAILABLE_DEFAULT or not GENERATIVE_POLISH_ENABLED:
-        return False
-    try:
-        import transformers  # noqa: F401
-        import torch  # noqa: F401
-
-        return True
-    except Exception:
-        return False
+    return False
 
 
 def generative_available() -> bool:
-    if not generative_package_present():
-        return False
-    if _failed:
-        return False
-    if _model is not None:
-        return True
-    try:
-        _ensure_model()
-        return _model is not None
-    except Exception:
-        return False
+    return False
 
 
 def backend_name() -> str | None:
-    _ensure_model()
-    return _backend
+    return None
 
 
 def backend_kind() -> str:
-    """Configured backend kind: seq2seq or causal (even before model load)."""
-    return GENERATIVE_BACKEND if GENERATIVE_BACKEND in {"seq2seq", "causal"} else "seq2seq"
-
-
-def _ensure_model():
-    global _tokenizer, _model, _failed, _backend, _backend_kind
-    if _model is not None or _failed:
-        return _model
-    with _lock:
-        if _model is not None or _failed:
-            return _model
-        try:
-            import torch
-            from transformers import AutoTokenizer
-
-            name = GENERATIVE_MODEL
-            kind = backend_kind()
-            logger.info(
-                "Loading generative rewrite model %s (backend=%s) …", name, kind
-            )
-            _tokenizer = AutoTokenizer.from_pretrained(name)
-            if _tokenizer.pad_token is None and _tokenizer.eos_token is not None:
-                _tokenizer.pad_token = _tokenizer.eos_token
-
-            if kind == "causal":
-                from transformers import AutoModelForCausalLM
-
-                _model = AutoModelForCausalLM.from_pretrained(name)
-                _backend_kind = "causal"
-            else:
-                from transformers import AutoModelForSeq2SeqLM
-
-                _model = AutoModelForSeq2SeqLM.from_pretrained(name)
-                _backend_kind = "seq2seq"
-
-            _model.eval()
-            _model.to("cpu")
-            _backend = f"transformers:{kind}:{name}"
-            _ = torch.__version__
-            logger.info("Generative rewrite ready (%s)", _backend)
-            return _model
-        except Exception as exc:
-            logger.warning("Generative rewrite unavailable: %s", exc)
-            _failed = True
-            _tokenizer = None
-            _model = None
-            _backend = None
-            _backend_kind = None
-            return None
-
-
-def _clean_gen_text(text: str) -> str:
-    text = re.sub(r"\s+", " ", (text or "").strip())
-    if not text:
-        return ""
-    # Drop casual chat wrappers
-    text = re.sub(
-        r"^(?:assistant|system|user)\s*[:\-]\s*", "", text, flags=re.I
-    ).strip()
-    low = text.lower()
-    for prefix in (
-        "rewrite:",
-        "paraphrase:",
-        "new paraphrase:",
-        "rewritten text:",
-        "here is the rewrite:",
-        "here is the rewritten text:",
-        "input:",
-        "paragraph:",
-        "sentence:",
-        "output:",
-    ):
-        if low.startswith(prefix):
-            text = text[len(prefix) :].strip()
-            low = text.lower()
-    # Strip surrounding quotes if the whole output is quoted
-    if len(text) >= 2 and text[0] in "\"'" and text[-1] == text[0]:
-        text = text[1:-1].strip()
-    return text
-
-
-def _surface_sim(a: str, b: str) -> float:
-    return SequenceMatcher(None, a.lower().strip(), b.lower().strip()).ratio()
-
-
-def _is_near_copy(original: str, candidate: str, max_sim: float) -> bool:
-    if not candidate or not candidate.strip():
-        return True
-    if candidate.strip().lower() == original.strip().lower():
-        return True
-    return _surface_sim(original, candidate) >= max_sim
-
-
-def _gen_kwargs(strength: int, *, n_return: int | None = None) -> dict[str, Any]:
-    strength = max(0, min(2, int(strength)))
-    base = {
-        0: {"temperature": 0.85, "top_p": 0.92, "top_k": 50, "max_new_tokens": 192, "n": 4},
-        1: {"temperature": 0.95, "top_p": 0.93, "top_k": 55, "max_new_tokens": 256, "n": 5},
-        2: {"temperature": 1.05, "top_p": 0.95, "top_k": 60, "max_new_tokens": 320, "n": 5},
-    }[strength]
-    n = n_return if n_return is not None else base["n"]
-    # Causal models: fewer return sequences to limit CPU RAM
-    if _backend_kind == "causal":
-        n = min(int(n), 3)
-    return {
-        "do_sample": True,
-        "num_beams": 1,
-        "temperature": base["temperature"],
-        "top_p": base["top_p"],
-        "top_k": base["top_k"],
-        "max_new_tokens": base["max_new_tokens"],
-        "no_repeat_ngram_size": 3,
-        "num_return_sequences": max(2, min(int(n), 5)),
-    }
-
-
-def _build_causal_prompt(text: str, tone: str, *, retry: bool = False) -> str:
-    tone_l = normalize_tone(tone)
-    user = (_CAUSAL_RETRY_USER if retry else _CAUSAL_USER).format(
-        tone=tone_l, text=text.strip()
-    )
-    messages = [
-        {"role": "system", "content": _SYSTEM_REWRITE},
-        {"role": "user", "content": user},
-    ]
-    assert _tokenizer is not None
-    try:
-        return _tokenizer.apply_chat_template(
-            messages,
-            tokenize=False,
-            add_generation_prompt=True,
-        )
-    except Exception:
-        # Fallback if chat template missing
-        return (
-            f"System: {_SYSTEM_REWRITE}\n\n"
-            f"User: {user}\n\n"
-            f"Assistant:"
-        )
-
-
-def _generate_seq2seq(prompt: str, strength: int, *, n_return: int | None = None) -> list[str]:
-    import torch
-
-    assert _model is not None and _tokenizer is not None
-    inputs = _tokenizer(
-        prompt,
-        return_tensors="pt",
-        truncation=True,
-        max_length=512,
-    )
-    kwargs = _gen_kwargs(strength, n_return=n_return)
-    with torch.no_grad():
-        out_ids = _model.generate(**inputs, **kwargs)
-    return [_tokenizer.decode(row, skip_special_tokens=True) for row in out_ids]
-
-
-def _generate_causal(prompt: str, strength: int, *, n_return: int | None = None) -> list[str]:
-    import torch
-
-    assert _model is not None and _tokenizer is not None
-    inputs = _tokenizer(
-        prompt,
-        return_tensors="pt",
-        truncation=True,
-        max_length=1024,
-    )
-    input_len = int(inputs["input_ids"].shape[-1])
-    kwargs = _gen_kwargs(strength, n_return=n_return)
-    # Avoid pad/eos issues on small causal models
-    if _tokenizer.eos_token_id is not None:
-        kwargs["eos_token_id"] = _tokenizer.eos_token_id
-    if _tokenizer.pad_token_id is not None:
-        kwargs["pad_token_id"] = _tokenizer.pad_token_id
-    with torch.no_grad():
-        out_ids = _model.generate(**inputs, **kwargs)
-    decoded: list[str] = []
-    for row in out_ids:
-        new_tokens = row[input_len:]
-        decoded.append(_tokenizer.decode(new_tokens, skip_special_tokens=True))
-    return decoded
-
-
-def _generate_once(
-    prompt: str,
-    strength: int,
-    *,
-    n_return: int | None = None,
-    causal: bool | None = None,
-) -> list[str]:
-    model = _ensure_model()
-    if model is None or _tokenizer is None:
-        return []
-    use_causal = (_backend_kind == "causal") if causal is None else causal
-    try:
-        if use_causal:
-            return _generate_causal(prompt, strength, n_return=n_return)
-        return _generate_seq2seq(prompt, strength, n_return=n_return)
-    except Exception as exc:
-        logger.warning("Generative decode failed: %s", exc)
-        return []
-
-
-def _chunk_long_paragraph(para: str, max_words: int = 180) -> list[str]:
-    """Split oversized paragraphs into sentence groups for model context limits."""
-    words = para.split()
-    if len(words) <= max_words:
-        return [para]
-    parts = re.split(r"(?<=[.!?])\s+", para.strip())
-    chunks: list[str] = []
-    buf: list[str] = []
-    count = 0
-    for sent in parts:
-        w = len(sent.split())
-        if buf and count + w > max_words:
-            chunks.append(" ".join(buf))
-            buf = [sent]
-            count = w
-        else:
-            buf.append(sent)
-            count += w
-    if buf:
-        chunks.append(" ".join(buf))
-    return chunks or [para]
-
-
-def rewrite_unit(
-    text: str,
-    *,
-    tone: str = "Neutral",
-    strength: int = 1,
-    min_meaning: float = 0.80,
-    max_sim: float = NEAR_COPY_SURFACE,
-) -> str:
-    """
-    Public targeted-unit rewrite (sentence or short paragraph).
-
-    Backend-agnostic: uses seq2seq or causal depending on GENERATIVE_BACKEND.
-    Returns the best validated rewrite, or the original unit if generation fails.
-    """
-    return _paraphrase_paragraph(
-        text,
-        tone,
-        strength,
-        min_meaning=min_meaning,
-        max_sim=max_sim,
-    )
-
-
-def _paraphrase_paragraph(
-    paragraph: str,
-    tone: str,
-    strength: int,
-    *,
-    min_meaning: float = 0.80,
-    max_sim: float = NEAR_COPY_SURFACE,
-) -> str:
-    """Generate candidates for one unit; return best validated rewrite or original."""
-    model = _ensure_model()
-    if model is None or _tokenizer is None:
-        return paragraph
-    if len(paragraph.split()) < 3:
-        return paragraph
-
-    tone_l = normalize_tone(tone)
-    if _backend_kind == "causal":
-        prompt = _build_causal_prompt(paragraph, tone_l, retry=False)
-    else:
-        prompt = _TONE_PROMPTS.get(tone_l, _TONE_PROMPTS["Neutral"]).format(
-            text=paragraph.strip()
-        )
-    raw = _generate_once(prompt, strength)
-    cands = [_clean_gen_text(x) for x in raw]
-
-    picked = pick_best_candidate(
-        paragraph,
-        cands,
-        tone=tone_l,
-        min_meaning=min_meaning,
-        max_surface=max_sim,
-        fallback=paragraph,
-    )
-
-    if _is_near_copy(paragraph, picked, max_sim):
-        if _backend_kind == "causal":
-            retry_prompt = _build_causal_prompt(paragraph, tone_l, retry=True)
-        else:
-            retry_prompt = _RETRY_PROMPT.format(
-                tone=tone_l.lower(), text=paragraph.strip()
-            )
-        raw2 = _generate_once(retry_prompt, min(2, strength + 1), n_return=5)
-        cands2 = [_clean_gen_text(x) for x in raw2]
-        picked2 = pick_best_candidate(
-            paragraph,
-            cands + cands2,
-            tone=tone_l,
-            min_meaning=max(0.76, min_meaning - 0.02),
-            max_surface=max_sim,
-            fallback=paragraph,
-        )
-        if not _is_near_copy(paragraph, picked2, max_sim):
-            picked = picked2
-        else:
-            logger.info(
-                "Generative unit still near-copy (sim=%.3f); keeping source chunk",
-                _surface_sim(paragraph, picked2),
-            )
-            return paragraph
-
-    return picked
-
-
-def generative_paraphrase(
-    text: str,
-    *,
-    tone: str = "Neutral",
-    strength: int = 1,
-    max_sim: float = NEAR_COPY_SURFACE,
-    min_meaning: float = 0.80,
-) -> str | None:
-    """
-    Quality-first paragraph paraphrase.
-    Returns None if the model is unavailable.
-    Near-copy paragraphs are left unchanged so the orchestrator can fall back.
-    """
-    if not text.strip():
-        return None
-    if _ensure_model() is None:
-        return None
+    return "none"
 
-    tone = normalize_tone(tone)
-    strength = max(0, min(2, int(strength)))
-    gen_strength = max(1, strength)
 
-    paras = split_paragraphs(text)
-    out_paras: list[str] = []
-    budget = int(GENERATIVE_MAX_PARAGRAPHS)
-    unlimited = budget <= 0
-    used = 0
-    changed_n = 0
+def rewrite_unit(*_args: Any, **_kwargs: Any) -> str | None:
+    return None
 
-    for para in paras:
-        if not para.strip():
-            continue
-        chunks = _chunk_long_paragraph(para)
-        new_chunks: list[str] = []
-        for chunk in chunks:
-            if not unlimited and used >= budget:
-                new_chunks.append(chunk)
-                continue
-            nxt = _paraphrase_paragraph(
-                chunk,
-                tone,
-                gen_strength,
-                min_meaning=min_meaning,
-                max_sim=max_sim,
-            )
-            used += 1
-            if not _is_near_copy(chunk, nxt, max_sim):
-                changed_n += 1
-            new_chunks.append(nxt)
-        out_paras.append(" ".join(new_chunks))
 
-    if not out_paras:
-        return None
-    result = "\n\n".join(p for p in out_paras if p)
-    logger.info(
-        "Generative paraphrase: %s/%s units changed (strength=%s, model=%s, "
-        "backend=%s, budget=%s, max_sim=%.2f)",
-        changed_n,
-        used,
-        gen_strength,
-        GENERATIVE_MODEL,
-        _backend_kind or backend_kind(),
-        "all" if unlimited else budget,
-        max_sim,
-    )
-    if used > 0 and changed_n == 0:
-        logger.info("Generative produced no useful paraphrases; returning near-copy for fallback")
-    return result
+def generative_paraphrase(*_args: Any, **_kwargs: Any) -> str | None:
+    return None
 
 
 def warm_generative() -> bool:
-    try:
-        ok = generative_available()
-        if ok:
-            rewrite_unit(
-                "The system works well for most users and saves time.",
-                tone="Neutral",
-                strength=1,
-            )
-        return ok
-    except Exception as exc:
-        logger.warning("Generative warm failed: %s", exc)
-        return False
+    return False

app/pipeline/orchestrator.py

@@ -1,64 +1,14 @@
-"""Orchestrate rewrite: classical / generative / hybrid (SmolLM2 or FLAN patch).
-
-Modes (PIPELINE_MODE):
-  classical  — rules + lexicon only
-  generative — whole-document generative rewrite
-  hybrid     — classical baseline → validate units → generative patch failed units
-"""
+"""Orchestrate rewrite via the structural engine (compatibility shim)."""
 
 from __future__ import annotations
 
 import logging
-import random
-import re
 import time
 from dataclasses import dataclass, field
-from difflib import SequenceMatcher
 
-from app.config import (
-    GENERATIVE_BACKEND,
-    GENERATIVE_MAX_PARAGRAPHS,
-    GENERATIVE_MODEL,
-    GENERATIVE_PRIMARY,
-    GRAMMAR_FIX_INPUT,
-    GRAMMAR_FIX_OUTPUT,
-    LEXICON_FALLBACK,
-    PIPELINE_MODE,
-)
-from app.pipeline.alignment import align_documents, iter_source_units
-from app.pipeline.candidate_ranker import NEAR_COPY_SURFACE
-from app.pipeline.candidate_validator import validate_candidate
-from app.pipeline.generative import (
-    backend_kind,
-    generative_available,
-    generative_paraphrase,
-    rewrite_unit,
-)
-from app.pipeline.grammar_fix import correct_text
-from app.pipeline.mechanics import enforce_length_budget, scrub_phrases, tidy
-from app.pipeline.ml_context import (
-    gen_was_used,
-    mark_gen_used,
-    ml_was_used,
-    set_ml_polish,
-)
-from app.pipeline.ml_polish import apply_minilm_polish
-from app.pipeline.minilm import minilm_available
-from app.pipeline.nlp import get_nlp, spacy_available
-from app.pipeline.normalize import (
-    normalize_whitespace,
-    split_paragraphs,
-    split_sentences_regex,
-    word_count,
-)
-from app.pipeline.restructure import restructure_sentence
+from app.engine.orchestrator import rewrite_document
+from app.pipeline.normalize import word_count
 from app.pipeline.similarity import SimilarityResult, compare_similarity
-from app.pipeline.synonym import rewrite_sentence_synonyms
-from app.pipeline.syntax_rewrite import (
-    apply_tone_contractions,
-    rewrite_paragraph_structure,
-)
-from app.pipeline.tone_style import apply_tone_style
 from app.pipeline.tones import normalize_tone
 
 logger = logging.getLogger("plainrewrite")
@@ -68,6 +18,8 @@ if not logger.handlers:
 
 @dataclass
 class HybridStats:
+    """Legacy stats shape kept for API compatibility."""
+
     units: int = 0
     classical_kept: int = 0
     regenerated: int = 0
@@ -92,1187 +44,12 @@ class RewriteResult:
     tone: str
     changed: bool
     notes: str = ""
-    pipeline_mode: str = ""
+    pipeline_mode: str = "structural"
     hybrid: HybridStats | None = None
     similarity: float = 1.0
-
-
-def _rng_for(text: str) -> random.Random:
-    return random.Random(sum(ord(c) for c in text) * 2654435761 & 0xFFFFFFFF)
-
-
-def _similarity_ratio(a: str, b: str) -> float:
-    return SequenceMatcher(None, a.lower(), b.lower()).ratio()
-
-
-def _light_post_gen(text: str, tone: str) -> str:
-    """After generative rewrite: scrub fillers + contractions only — no synonym thrash."""
-    out = scrub_phrases(text)
-    out = apply_tone_contractions(out, tone)
-    return tidy(out)
-
-
-def _ensure_unit_terminal(text: str) -> str:
-    """Guarantee a sentence terminator so units stay countable after join."""
-    s = (text or "").strip()
-    if not s:
-        return s
-    if s[-1] not in ".!?":
-        s += "."
-    return s
-
-
-def _unit_stems(text: str) -> set[str]:
-    """Content stems shared with the validator (for lexical-novelty checks)."""
-    from app.pipeline.candidate_validator import _content_tokens
-
-    return _content_tokens(text or "")
-
-
-def _stem_keep_ratio(source: str, candidate: str) -> float:
-    src = _unit_stems(source)
-    if not src:
-        return 1.0
-    return len(src & _unit_stems(candidate)) / len(src)
-
-
-def _stem_jaccard(source: str, candidate: str) -> float:
-    a, b = _unit_stems(source), _unit_stems(candidate)
-    if not a and not b:
-        return 1.0
-    return len(a & b) / max(1, len(a | b))
-
-
-def _paraphrase_hard_ok(source_unit: str, trial: str) -> bool:
-    """Meaning safety for forced rewrites.
-
-    Allows real synonym/phrase changes (relaxed key_content/collocation) but still
-    blocks polarity flips, invention, topic swaps, etc.
-    """
-    t = (trial or "").strip()
-    if not t:
-        return False
-    t = _clamp_to_source_shape(source_unit, t) or t
-    v = validate_candidate(source_unit, t, min_meaning=0.62, max_surface=0.995)
-    # key_content/coverage/collocation/drift are too strict for synonym paraphrases;
-    # enforce a softer stem floor instead.
-    soft = {
-        "too_similar",
-        "identical",
-        "near_copy",
-        "too_divergent",
-        "key_content",
-        "coverage",
-        "drift",
-        "grammar_worse",
-        "collocation",
-    }
-    hard_keys = [r.split(":")[0] for r in v.reasons if r.split(":")[0] not in soft]
-    if _hard_fail(hard_keys):
-        return False
-    if _stem_keep_ratio(source_unit, t) < 0.30:
-        return False
-    return True
-
-
-def _content_word_bag(text: str) -> frozenset[str]:
-    """Lowercased content words (len≥4)."""
-    return frozenset(
-        w.strip("'")
-        for w in re.findall(r"[a-zA-Z']+", (text or "").lower())
-        if len(w.strip("'")) >= 4
-    )
-
-
-def _token_seq(text: str) -> tuple[str, ...]:
-    return tuple(re.findall(r"[a-zA-Z']+", (text or "").lower()))
-
-
-def _is_structural_reorder(source: str, candidate: str) -> bool:
-    """Same content words, different order (user-expected sentence reorder)."""
-    src_bag = _content_word_bag(source)
-    cand_bag = _content_word_bag(candidate)
-    if not src_bag or src_bag != cand_bag:
-        return False
-    return _token_seq(source) != _token_seq(candidate)
-
-
-def _is_true_rewrite(source_unit: str, candidate: str) -> bool:
-    """Accept structural reorder OR lexical wording change — not identity."""
-    src = (source_unit or "").strip()
-    cand = (candidate or "").strip()
-    if not src or not cand:
-        return False
-    if cand.lower().rstrip(".!?") == src.lower().rstrip(".!?"):
-        return False
-    if not _paraphrase_hard_ok(src, cand):
-        return False
-    sim = _similarity_ratio(src, cand)
-    # Preferred path: reorder same words into a new structure
-    if _is_structural_reorder(src, cand):
-        return sim < 0.995
-    # Lexical path: different content words
-    jacc = _stem_jaccard(src, cand)
-    if jacc >= 0.92:
-        return False
-    if sim > 0.90:
-        return False
-    return True
-
-
-def _safe_paraphrase_trial(source_unit: str, trial: str) -> tuple[bool, float]:
-    """Return (is_true_rewrite, surface_sim)."""
-    t = (trial or "").strip()
-    if not t:
-        return False, 1.0
-    t = _ensure_unit_terminal(_clamp_to_source_shape(source_unit, t) or t)
-    if not _is_true_rewrite(source_unit, t):
-        return False, 1.0
-    return True, _similarity_ratio(source_unit, t)
-
-
-def _rewrite_score(source_unit: str, candidate: str) -> float:
-    """Prefer structural reorder; otherwise reward lexical difference."""
-    sim = _similarity_ratio(source_unit, candidate)
-    if _is_structural_reorder(source_unit, candidate):
-        # Strong preference for clear reorders (user expectation)
-        return 1.25 + (1.0 - sim)
-    jacc = _stem_jaccard(source_unit, candidate)
-    return (1.0 - sim) + 0.85 * (1.0 - jacc)
-
-
-# Phrase-level meaning-preserving rewrites (change wording, not just order).
-# Prefer swaps that keep enough claim stems to stay meaning-safe.
-_PHRASE_REWRITES: tuple[tuple[re.Pattern[str], str], ...] = (
-    (re.compile(r"\bget(?:s)?\s+diseases\b", re.I), "get sick"),
-    (re.compile(r"\bgetting\s+diseases\b", re.I), "getting sick"),
-    (
-        re.compile(r"\bless chance to gets?\s+diseases\b", re.I),
-        "less chance to get sick",
-    ),
-    (
-        re.compile(r"\bless chance of getting\s+diseases\b", re.I),
-        "less chance of getting sick",
-    ),
-    (re.compile(r"\bwatching television\b", re.I), "watching TV"),
-    (re.compile(r"\bdoing physical activities\b", re.I), "working out"),
-    (re.compile(r"\bphysical activities\b", re.I), "real exercise"),
-    (re.compile(r"\bexercise regularly\b", re.I), "work out often"),
-    (re.compile(r"\bexercises regularly\b", re.I), "works out often"),
-    (re.compile(r"\ba person exercise\b", re.I), "a person works out"),
-    (re.compile(r"\ba person exercises\b", re.I), "a person works out"),
-    (re.compile(r"\bprefers watching\b", re.I), "would rather watch"),
-    (re.compile(r"\bprefer watching\b", re.I), "would rather watch"),
-    (re.compile(r"\bmany peoples\b", re.I), "a lot of people"),
-    (re.compile(r"\bmany people\b", re.I), "a lot of people"),
-    (re.compile(r"\bUnfortunately,\s*", re.I), "Sadly, "),
-    (
-        re.compile(
-            r"\bis one of the most important\s+\w+\s+that keeps our body fit\b",
-            re.I,
-        ),
-        "is a key way to keep the body fit",
-    ),
-    (
-        re.compile(r"\bis one of the most important\b", re.I),
-        "is a key",
-    ),
-    (re.compile(r"\bkeeps our body fit\b", re.I), "helps the body stay fit"),
-    (re.compile(r"\bfind it harder to\b", re.I), "struggle more to"),
-    (re.compile(r"\binstead of\b", re.I), "rather than"),
-)
-
-
-def _sentence_reorder_variants(text: str) -> list[str]:
-    """Structural reorders that keep the same words (user-expected style).
-
-    Example:
-      Ram went to school yesterday happily.
-      → Yesterday, Ram happily went to school.
-    """
-    out: list[str] = []
-    base = (text or "").strip()
-    if not base:
-        return out
-    end_m = re.search(r"[.!?]+$", base)
-    end = end_m.group(0) if end_m else "."
-    core = base[: end_m.start()] if end_m else base
-
-    # Subj + motion-verb + to + place + time + manner-ly
-    m = re.match(
-        r"^(?P<subj>[A-Za-z][\w'-]*)\s+"
-        r"(?P<verb>went|go|goes|walked|ran|drove|came|moved)\s+to\s+"
-        r"(?P<place>.+?)\s+"
-        r"(?P<time>yesterday|today|tomorrow|earlier|later|recently)\s+"
-        r"(?P<manner>\w+ly)$",
-        core,
-        flags=re.I,
-    )
-    if m:
-        subj = m.group("subj")
-        verb = m.group("verb")
-        place = m.group("place").strip()
-        time = m.group("time")
-        manner = m.group("manner")
-        # Preferred: Yesterday, Ram happily went to school.
-        out.append(
-            f"{_cap_first(time)}, {subj} {manner} {verb.lower()} to {place}{end}"
-        )
-        out.append(
-            f"{_cap_first(time)}, {subj} {verb.lower()} to {place} {manner}{end}"
-        )
-        out.append(f"{subj} {manner} {verb.lower()} to {place} {time}{end}")
-
-    # Trailing time → front: "... yesterday."
-    m = re.match(
-        r"^(?P<main>.+?)\s+"
-        r"(?P<time>yesterday|today|tomorrow|earlier|later|recently|now)"
-        r"$",
-        core,
-        flags=re.I,
-    )
-    if m and len(m.group("main").split()) >= 3:
-        main = m.group("main").rstrip(" ,")
-        time = m.group("time")
-        out.append(f"{_cap_first(time)}, {_lower_first(main)}{end}")
-
-    # Trailing manner -ly → before main verb (simple Subj ... ly)
-    m = re.match(
-        r"^(?P<subj>[A-Za-z][\w'-]*)\s+(?P<mid>.+?)\s+(?P<manner>\w+ly)$",
-        core,
-        flags=re.I,
-    )
-    if m and " to " in f" {m.group('mid')} ":
-        subj, mid, manner = m.group("subj"), m.group("mid").strip(), m.group("manner")
-        # Insert manner after subject
-        out.append(f"{subj} {manner} {mid}{end}")
-
-    # "Y instead of X" → "Instead of X, Y"
-    m = re.match(
-        r"^(?P<main>.+?)\s+instead of\s+(?P<alt>.+)$",
-        core,
-        flags=re.I,
-    )
-    if m:
-        main = m.group("main").rstrip(" ,")
-        alt = m.group("alt").rstrip(" .!?")
-        out.append(f"Instead of {alt}, {_lower_first(main)}{end}")
-
-    # "X because Y" → "Because Y, X"
-    m = re.match(
-        r"^(?P<main>.+?)\s+because\s+(?P<reason>.+)$",
-        core,
-        flags=re.I,
-    )
-    if m and len(m.group("main").split()) >= 3:
-        main = m.group("main").rstrip(" ,")
-        reason = m.group("reason").rstrip(" .!?")
-        out.append(f"Because {reason}, {_lower_first(main)}{end}")
-
-    # Dedup
-    seen: set[str] = set()
-    uniq: list[str] = []
-    for t in out:
-        key = t.strip().lower()
-        if not key or key in seen:
-            continue
-        seen.add(key)
-        uniq.append(t.strip())
-    return uniq
-
-
-def _apply_phrase_rewrites(text: str, *, max_swaps: int = 4) -> list[str]:
-    """Build increasingly rewritten phrase variants."""
-    out: list[str] = []
-    base = (text or "").strip()
-    if not base:
-        return out
-    # Single-swap variants
-    for pat, repl in _PHRASE_REWRITES:
-        if not pat.search(base):
-            continue
-        cand = pat.sub(repl, base, count=1)
-        if cand.strip() and cand.strip() != base:
-            out.append(cand.strip())
-    # Stacked multi-swap
-    stacked = base
-    n = 0
-    for pat, repl in _PHRASE_REWRITES:
-        if n >= max_swaps:
-            break
-        if pat.search(stacked):
-            nxt = pat.sub(repl, stacked, count=1)
-            if nxt != stacked:
-                stacked = nxt
-                n += 1
-                out.append(stacked.strip())
-    return out
-
-
-def _esl_structural_variants(text: str) -> list[str]:
-    """Lexical ESL rewrites (must change wording — not mere reorder)."""
-    out: list[str] = []
-    base = (text or "").strip()
-    if not base:
-        return out
-    end_m = re.search(r"[.!?]+$", base)
-    end = end_m.group(0) if end_m else "."
-    core = base[: end_m.start()] if end_m else base
-
-    m = re.match(
-        r"^(?:Another|One|An)\s+important\s+thing\s+(?:is|are)\s+(.+)$",
-        core,
-        flags=re.I,
-    )
-    if m:
-        topic = m.group(1).strip().rstrip(".!?")
-        if topic:
-            # Keep topic head "thing" so topic_anchor passes; change other wording
-            out.append(f"{_cap_first(topic)} remains an important thing{end}")
-            out.append(f"{_cap_first(topic)} is still an important thing{end}")
-            out.append(f"A further important thing is {_lower_first(topic)}{end}")
-
-    m = re.match(r"^Without enough\s+(\w+),\s*(.+)$", core, flags=re.I)
-    if m:
-        noun, rest = m.group(1), m.group(2).strip()
-        if noun and rest:
-            out.append(f"When {noun} runs short, {_lower_first(rest)}{end}")
-            out.append(f"If {noun} is lacking, {_lower_first(rest)}{end}")
-            out.append(f"Too little {noun} leaves {_lower_first(rest)}{end}")
-
-    m = re.match(
-        r"^(.+?)\s+is one of the most important\s+(.+)$",
-        core,
-        flags=re.I,
-    )
-    if m:
-        subj, rest = m.group(1).strip(), m.group(2).strip()
-        if subj and rest:
-            out.append(f"{_cap_first(subj)} is a key part of {rest}{end}")
-            out.append(f"{_cap_first(subj)} matters a lot for {rest}{end}")
-
-    m = re.match(r"^It is important (?:that|to)\s+(.+)$", core, flags=re.I)
-    if m:
-        rest = m.group(1).strip()
-        if rest:
-            out.append(f"People need to {_lower_first(rest)}{end}")
-            out.append(f"{_cap_first(rest)} really matters{end}")
-
-    # If a person X, they will Y → People who X will Y
-    m = re.match(
-        r"^(?:If\s+)?a person\s+(.+?),\s*they will\s+(.+)$",
-        core,
-        flags=re.I,
-    )
-    if m:
-        act, result = m.group(1).strip(), m.group(2).strip()
-        out.append(f"People who {act} will {result}{end}")
-        out.append(f"Anyone who {act} will {result}{end}")
-
-    return out
-
-
-def _cap_first(text: str) -> str:
-    t = (text or "").strip()
-    if not t:
-        return t
-    return t[0].upper() + t[1:]
-
-
-def _lower_first(text: str) -> str:
-    t = (text or "").strip()
-    if not t:
-        return t
-    return t[0].lower() + t[1:]
-
-
-_SAFE_SURFACE_SWAPS: tuple[tuple[re.Pattern[str], str], ...] = (
-    (re.compile(r"\bIn addition,\b", re.I), "Also,"),
-    (re.compile(r"\bAdditionally,\b", re.I), "Also,"),
-    (re.compile(r"\bHowever,\b", re.I), "Still,"),
-    (re.compile(r"\bTherefore,\b", re.I), "So,"),
-    (re.compile(r"\bDue to the fact that\b", re.I), "Because"),
-    (re.compile(r"\bin order to\b", re.I), "to"),
-    (re.compile(r"\ba large number of\b", re.I), "many"),
-    (re.compile(r"\bMany\b"), "A lot of"),
-    (re.compile(r"\bis able to\b", re.I), "can"),
-    (re.compile(r"\bare able to\b", re.I), "can"),
-    (re.compile(r"\bPeople should\b"), "People need to"),
-    (re.compile(r"\bpeople should\b"), "people need to"),
-    (re.compile(r"\bbecause of\b", re.I), "due to"),
-    (re.compile(r"\bfor example\b", re.I), "for instance"),
-    (re.compile(r"\bthink that\b", re.I), "think"),
-    (re.compile(r"\bbelieve that\b", re.I), "believe"),
-    (re.compile(r"\bby giving\b", re.I), "by offering"),
-    (re.compile(r"\bhelp communities\b", re.I), "support communities"),
-)
-
-_VISIBLE_TARGET_SIM = 0.82
-_LM_SHIP_MAX_SIM = 0.85
-
-
-def _surface_humanize_variants(text: str, rng: random.Random) -> list[str]:
-    """Reorder + phrase + light surface variants."""
-    out: list[str] = []
-    base = (text or "").strip()
-    if not base:
-        return out
-
-    out.extend(_sentence_reorder_variants(base))
-    out.extend(_apply_phrase_rewrites(base))
-    out.extend(_esl_structural_variants(base))
-
-    # Because-clause fronting only when combined with a phrase rewrite later
-    m = re.match(
-        r"^(?P<main>.+?)\s+because\s+(?P<reason>.+?)(?P<end>[.!?])?$",
-        base,
-        flags=re.I,
-    )
-    if m and len(m.group("main").split()) >= 3:
-        main = m.group("main").rstrip(" ,")
-        reason = m.group("reason").rstrip(" .!?")
-        end = m.group("end") or "."
-        out.append(f"Because {reason}, {_lower_first(main)}{end}")
-
-    swaps = list(_SAFE_SURFACE_SWAPS)
-    rng.shuffle(swaps)
-    for pat, repl in swaps:
-        if not pat.search(base):
-            continue
-        cand = pat.sub(repl, base, count=1)
-        if cand.strip() and cand.strip() != base:
-            out.append(cand.strip())
-    # Stack phrase rewrites on surface swaps
-    for seed in list(out[:8]):
-        out.extend(_apply_phrase_rewrites(seed))
-        out.extend(_esl_structural_variants(seed))
-    return out
-
-
-def _visible_unit_paraphrase(
-    source_unit: str,
-    tone: str,
-    rng: random.Random,
-    *,
-    strength: int = 2,
-    target_sim: float = _VISIBLE_TARGET_SIM,
-) -> str:
-    """Prefer dependency/template reorder; fall back to classical paraphrase."""
-    base = scrub_phrases(source_unit)
-    base = apply_tone_contractions(base, tone)
-    base = tidy(correct_text(base) if base else "")
-    if not base.strip():
-        base = source_unit
-
-    # --- Primary: dep+template restructure (no synonym thrash) ---
-    try:
-        rr = restructure_sentence(base, min_confidence=0.55)
-        if rr and rr.text.strip():
-            cand = tidy(correct_text(rr.text) if rr.text else rr.text)
-            cand = _ensure_unit_terminal(
-                _clamp_to_source_shape(source_unit, cand) or cand
-            )
-            if _is_true_rewrite(source_unit, cand) or _is_structural_reorder(
-                source_unit, cand
-            ):
-                if _paraphrase_hard_ok(source_unit, cand):
-                    return cand
-    except Exception:
-        pass
-
-    structured = base
-    try:
-        structured = rewrite_paragraph_structure(base, strength, rng, tone=tone)
-        structured = tidy(structured)
-        clamped = _clamp_to_source_shape(source_unit, structured)
-        if clamped:
-            structured = clamped
-        elif _sentence_count_simple(structured) != _sentence_count_simple(source_unit):
-            structured = base
-    except Exception:
-        structured = base
-
-    trials: list[str] = []
-    # Prefer structural reorder first (user expectation)
-    for seed in (base, structured):
-        trials.extend(_sentence_reorder_variants(seed))
-    for seed in (base, structured):
-        trials.extend(_apply_phrase_rewrites(seed))
-        trials.extend(_esl_structural_variants(seed))
-        trials.extend(_surface_humanize_variants(seed, rng))
-    if structured.strip() and structured.strip() != base.strip():
-        trials.append(structured)
-
-    # Synonym passes — allow stronger lexicon; safety filter keeps meaning
-    for syn_src, syn_str, force in (
-        (structured, 2, True),
-        (base, 2, True),
-        (structured, 1, True),
-        (base, 1, False),
-    ):
-        try:
-            cand = rewrite_sentence_synonyms(
-                syn_src,
-                syn_str,
-                rng,
-                tone=tone,
-                force_all_lexicon=force,
-            )
-            cand = apply_tone_contractions(cand, tone)
-            cand = tidy(correct_text(cand) if cand else "")
-            if cand.strip():
-                trials.append(cand)
-                trials.extend(_apply_phrase_rewrites(cand))
-                trials.extend(_esl_structural_variants(cand))
-        except Exception:
-            continue
-
-    seen: set[str] = set()
-    uniq: list[str] = []
-    for t in trials:
-        key = t.strip().lower()
-        if not key or key in seen:
-            continue
-        seen.add(key)
-        uniq.append(t.strip())
-
-    best = _ensure_unit_terminal(source_unit)
-    best_score = -1.0
-    for trial in uniq:
-        ok, sim = _safe_paraphrase_trial(source_unit, trial)
-        if not ok:
-            continue
-        t = _ensure_unit_terminal(
-            _clamp_to_source_shape(source_unit, trial.strip()) or trial.strip()
-        )
-        score = _rewrite_score(source_unit, t)
-        if score > best_score or (
-            abs(score - best_score) < 1e-9 and sim < _similarity_ratio(source_unit, best)
-        ):
-            best = t
-            best_score = score
-
-    # Phrase polish only for lexical winners — never undo a structural reorder
-    if best_score >= 0 and not _is_structural_reorder(source_unit, best):
-        for polished in _apply_phrase_rewrites(best, max_swaps=4):
-            if not _is_true_rewrite(source_unit, polished):
-                continue
-            if _is_structural_reorder(source_unit, polished):
-                continue
-            t = _ensure_unit_terminal(
-                _clamp_to_source_shape(source_unit, polished.strip())
-                or polished.strip()
-            )
-            score = _rewrite_score(source_unit, t)
-            if score > best_score:
-                best = t
-                best_score = score
-    return best
-
-
-def _light_revert_polish(
-    source_unit: str,
-    tone: str,
-    rng: random.Random,
-    *,
-    strength: int = 0,
-) -> str:
-    """Backward-compatible name — now runs true lexical paraphrase."""
-    return _visible_unit_paraphrase(
-        source_unit,
-        tone,
-        rng,
-        strength=max(2, strength + 1),
-        target_sim=_VISIBLE_TARGET_SIM,
-    )
-
-
-def _try_restructure_unit(source_unit: str) -> str | None:
-    """Return a safe dep+template reorder, or None if unavailable."""
-    src = (source_unit or "").strip()
-    if not src or len(src.split()) < 3:
-        return None
-    try:
-        base = tidy(correct_text(src) if src else "") or src
-        rr = restructure_sentence(base, min_confidence=0.55)
-        if not rr or not rr.text.strip():
-            return None
-        cand = tidy(correct_text(rr.text) if rr.text else rr.text)
-        cand = _ensure_unit_terminal(
-            _clamp_to_source_shape(src, cand) or cand
-        )
-        if not (_is_structural_reorder(src, cand) or _is_true_rewrite(src, cand)):
-            return None
-        if not _paraphrase_hard_ok(src, cand):
-            return None
-        return cand
-    except Exception:
-        return None
-
-
-def _require_visible_rewrite(
-    source_unit: str,
-    candidate: str,
-    tone: str,
-    rng: random.Random,
-    *,
-    target_sim: float = _VISIBLE_TARGET_SIM,
-    min_words: int = 4,
-) -> str:
-    """Prefer structural reorder; otherwise force a visible true rewrite."""
-    src = (source_unit or "").strip()
-    cand = (candidate or src).strip() or src
-    if len(src.split()) < min_words:
-        return _ensure_unit_terminal(cand)
-
-    # Structural reorder is the preferred finished product — do not synonym-polish it
-    if _is_structural_reorder(src, cand) and _paraphrase_hard_ok(src, cand):
-        return _ensure_unit_terminal(cand)
-
-    if _is_true_rewrite(src, cand) and _similarity_ratio(src, cand) <= target_sim:
-        return _ensure_unit_terminal(cand)
-
-    # Prefer dep+template restructure before classical/LM thrash
-    restructured = _try_restructure_unit(src)
-    if restructured:
-        return restructured
-
-    # Classical true rewrite from source (ignore weak LM near-copies)
-    trials = [
-        _visible_unit_paraphrase(src, tone, rng, strength=2, target_sim=target_sim),
-        _visible_unit_paraphrase(src, tone, rng, strength=2, target_sim=0.78),
-    ]
-    best = _ensure_unit_terminal(cand)
-    best_score = -1.0
-    if _is_true_rewrite(src, best):
-        best_score = _rewrite_score(src, best)
-    for trial in trials:
-        if not _is_true_rewrite(src, trial):
-            continue
-        t = _ensure_unit_terminal(
-            _clamp_to_source_shape(src, trial.strip()) or trial.strip()
-        )
-        score = _rewrite_score(src, t)
-        if score > best_score:
-            best = t
-            best_score = score
-    return best
-
-
-def _boost_if_near_copy(
-    source_unit: str,
-    candidate: str,
-    tone: str,
-    rng: random.Random,
-    *,
-    threshold: float = _VISIBLE_TARGET_SIM,
-) -> str:
-    """If a kept unit is still nearly identical, force a true rewrite."""
-    return _require_visible_rewrite(
-        source_unit,
-        candidate,
-        tone,
-        rng,
-        target_sim=min(threshold, _VISIBLE_TARGET_SIM),
-    )
-
-
-def _assemble_hybrid_units(
-    source_units: list[tuple[int, str]],
-    outputs: list[str],
-) -> str:
-    """Join one output per source unit; never drop a unit."""
-    if len(outputs) != len(source_units):
-        # Repair: pad with source text
-        fixed = list(outputs)
-        while len(fixed) < len(source_units):
-            fixed.append(source_units[len(fixed)][1])
-        outputs = fixed[: len(source_units)]
-
-    by_para: dict[int, list[str]] = {}
-    for (para_idx, src_u), out in zip(source_units, outputs):
-        piece = _ensure_unit_terminal((out or src_u).strip() or src_u)
-        by_para.setdefault(para_idx, []).append(piece)
-    paras = [" ".join(by_para[i]).strip() for i in sorted(by_para.keys()) if by_para[i]]
-    return tidy("\n\n".join(p for p in paras if p))
-
-
-def _sentence_count_simple(text: str) -> int:
-    parts = re.split(r"[.!?]+", (text or "").strip())
-    return len([p for p in parts if p.strip()])
-
-
-def _clamp_to_source_shape(source_unit: str, candidate: str) -> str:
-    """Keep LM output to the same sentence count as the source unit (1→1)."""
-    c = (candidate or "").strip()
-    if not c:
-        return ""
-    o_n = _sentence_count_simple(source_unit)
-    c_n = _sentence_count_simple(c)
-    if o_n == 1 and c_n != 1:
-        # Take first sentence only; if still multi or empty, reject
-        first = re.split(r"(?<=[.!?])\s+", c, maxsplit=1)[0].strip()
-        if _sentence_count_simple(first) == 1 and first:
-            return first
-        return ""
-    if o_n >= 1 and c_n < o_n:
-        return ""
-    return c
-
-
-def _classical_paragraph(paragraph: str, tone: str, strength: int, rng: random.Random) -> str:
-    """Offline fallback: structure + curated lexicon (not the primary quality path)."""
-    styled = apply_tone_style(paragraph, tone, strength, rng)
-    structured = rewrite_paragraph_structure(styled, strength, rng, tone=tone)
-    nlp = get_nlp()
-    if nlp is not None:
-        sents = [s.text.strip() for s in nlp(structured).sents if s.text.strip()]
-    else:
-        sents = split_sentences_regex(structured)
-    rewritten_sents = [
-        rewrite_sentence_synonyms(s, strength, rng, tone=tone) for s in sents
-    ]
-    joined = " ".join(rewritten_sents)
-    joined = scrub_phrases(joined)
-    joined = apply_tone_style(joined, tone, strength, rng)
-    joined = apply_tone_contractions(joined, tone)
-    return joined.strip()
-
-
-def _classical_rewrite(
-    source: str,
-    tone: str,
-    strength: int,
-    rng: random.Random,
-    *,
-    force: bool = False,
-) -> str:
-    paragraphs = split_paragraphs(source)
-    out = [_classical_paragraph(p, tone, strength, rng) for p in paragraphs]
-    rewritten = tidy("\n\n".join(out))
-    rewritten = scrub_phrases(rewritten)
-    rewritten = apply_tone_style(rewritten, tone, strength, rng)
-    rewritten = apply_tone_contractions(rewritten, tone)
-    if force:
-        bump = min(2, strength + 1)
-        paras = split_paragraphs(rewritten)
-        forced: list[str] = []
-        for para in paras:
-            bumped = apply_tone_style(para, tone, bump, rng)
-            bumped = rewrite_paragraph_structure(bumped, bump, rng, tone=tone)
-            nlp = get_nlp()
-            if nlp is not None:
-                sents = [s.text.strip() for s in nlp(bumped).sents if s.text.strip()]
-            else:
-                sents = split_sentences_regex(bumped)
-            rewritten_sents = [
-                rewrite_sentence_synonyms(
-                    s, bump, rng, tone=tone, force_all_lexicon=True
-                )
-                for s in sents
-            ]
-            joined = scrub_phrases(" ".join(rewritten_sents))
-            joined = apply_tone_contractions(joined, tone)
-            forced.append(joined.strip())
-        rewritten = tidy("\n\n".join(p for p in forced if p))
-    return rewritten
-
-
-def _hard_fail(reasons: list[str]) -> bool:
-    for r in reasons:
-        if r in {
-            "polarity",
-            "numbers",
-            "quotes",
-            "length",
-            "coverage",
-            "key_content",
-            "topic_anchor",
-            "entity_inject",
-            "drift",
-            "broken",
-            "grammar_worse",
-            "invention",
-            "collocation",
-            "shape",
-        }:
-            return True
-        if r.startswith("meaning") or r.startswith("entity"):
-            return True
-    return False
-
-
-def _needs_patch(
-    source_unit: str,
-    candidate: str,
-    *,
-    max_sim: float = 0.88,
-) -> tuple[bool, list[str]]:
-    """Return (needs_regen, reasons). Near-copy, weak rewrite, or validation failure → patch."""
-    if not candidate.strip():
-        return True, ["empty"]
-    if not source_unit.strip():
-        return False, []
-    # Stricter near-copy bar for hybrid so more units get generative paraphrase
-    v = validate_candidate(source_unit, candidate, min_meaning=0.80, max_surface=max_sim)
-    surf = v.surface_sim
-    reasons = list(v.reasons)
-    if v.ok and surf >= 0.88:
-        return True, ["weak_rewrite"]
-    if v.ok:
-        return False, []
-    return True, reasons
-
-
-def _safe_classical_unit(source_unit: str, candidate: str) -> str:
-    """Grammar-clean classical unit; revert to source on hard meaning failure."""
-    cleaned = tidy(correct_text(candidate) if candidate else "")
-    if not cleaned.strip():
-        return source_unit
-    v = validate_candidate(source_unit, cleaned, min_meaning=0.68, max_surface=0.99)
-    if _hard_fail(v.reasons):
-        return source_unit
-    return cleaned
-
-
-def _accept_generative(source: str, draft: str) -> bool:
-    """Whole-document gate: polarity/meaning must hold vs grammar-corrected source."""
-    if not draft or not draft.strip():
-        return False
-
-    s_paras = split_paragraphs(source)
-    d_paras = split_paragraphs(draft)
-    if len(s_paras) == len(d_paras) and s_paras:
-        for sp, dp in zip(s_paras, d_paras):
-            if sp.strip() == dp.strip():
-                continue
-            v = validate_candidate(sp, dp, min_meaning=0.78)
-            if not v.ok and _hard_fail(v.reasons):
-                logger.info("Generative para rejected: %s", v.reasons)
-                return False
-        return True
-    v = validate_candidate(source, draft, min_meaning=0.78)
-    if v.ok or "identical" in v.reasons:
-        return True
-    if _hard_fail(v.reasons):
-        logger.info("Generative doc rejected: %s", v.reasons)
-        return False
-    return True
-
-
-def _looks_needs_lm(text: str, strength: int) -> bool:
-    """At Normal+, send almost all claim units to LM (short phrases still skipped)."""
-    words = text.split()
-    if len(words) < 4:
-        return False
-    if strength <= 0:
-        return False
-    # Residual ESL / awkward patterns after grammar — always send to LM
-    low = text.lower()
-    residual = (
-        "peoples",
-        "enough times",
-        "don't realizes",
-        "does realizes",
-        "it affect ",
-        "to gets ",
-        "to obtains ",
-        "a person exercise ",
-        "foods are becoming",
-        "prefers watching",
-    )
-    if any(m in low for m in residual):
-        return True
-    # Normal+: rewrite short claims too (was ≥8 — caused grammar-only sleep example)
-    if strength >= 1 and len(words) >= 4:
-        return True
-    return strength >= 2
-
-
-def _accept_lm_unit(
-    source_unit: str, candidate: str, *, max_sim: float = _LM_SHIP_MAX_SIM
-) -> tuple[bool, list[str]]:
-    """Strict gate for a single LM-rewritten sentence — must be a true rewrite."""
-    if not candidate or not candidate.strip():
-        return False, ["empty"]
-    o_n = _sentence_count_simple(source_unit)
-    c_n = _sentence_count_simple(candidate)
-    if o_n == 1 and c_n != 1:
-        return False, ["shape"]
-    if o_n >= 1 and c_n < o_n:
-        return False, ["shape"]
-    sim = _similarity_ratio(source_unit, candidate)
-    if sim >= max_sim or sim >= _LM_SHIP_MAX_SIM:
-        return False, ["near_copy"]
-    # Must be a true lexical rewrite (not grammar/reorder)
-    if not _is_true_rewrite(source_unit, candidate):
-        return False, ["near_copy"]
-    v = validate_candidate(
-        source_unit, candidate, min_meaning=0.72, max_surface=max_sim
-    )
-    soft = {
-        "too_similar",
-        "identical",
-        "near_copy",
-        "too_divergent",
-        "key_content",
-        "coverage",
-        "drift",
-        "grammar_worse",
-        "collocation",
-    }
-    hard_keys = [r.split(":")[0] for r in v.reasons if r.split(":")[0] not in soft]
-    if _hard_fail(hard_keys):
-        return False, list(v.reasons) or ["reject"]
-    return True, []
-
-
-def _hybrid_rewrite(
-    source: str,
-    tone: str,
-    strength: int,
-    rng: random.Random,
-    *,
-    can_generate: bool,
-) -> tuple[str, HybridStats, bool]:
-    """
-    Reorder-first hybrid (dynamic documents):
-
-      grammar-corrected source
-        → dep+template restructure when confident (skip LM)
-        → skip clean/short units (latency)
-        → per-sentence Small LM rewrite for leftovers
-        → strict validate (shape/invention/coverage)
-        → accept | retry | classical visible rewrite  (1:1 unit invariant)
-
-    Classical lexicon is used when generative is unavailable
-    (and LEXICON_FALLBACK is on), still preferring restructure.
-
-    Returns (text, stats, used_any_gen).
-    """
-    stats = HybridStats()
-    used_gen = False
-
-    # --- Offline path: generative unavailable → classical lexicon fallback ---
-    if not can_generate:
-        if not LEXICON_FALLBACK:
-            stats.units = len(iter_source_units(source)) or 1
-            stats.reverted_source = stats.units
-            return source, stats, False
-        baseline = tidy(correct_text(_classical_rewrite(source, tone, strength, rng)))
-        units = align_documents(source, baseline)
-        by_para: dict[int, list[str]] = {}
-        for unit in units:
-            stats.units += 1
-            src_u = unit.source.strip()
-            cand_u = unit.candidate.strip()
-            if not src_u:
-                if cand_u:
-                    by_para.setdefault(unit.paragraph_index, []).append(cand_u)
-                continue
-            safe = _safe_classical_unit(src_u, cand_u)
-            if strength >= 1 and len(src_u.split()) >= 4:
-                safe = _require_visible_rewrite(
-                    src_u, safe, tone, rng, target_sim=_VISIBLE_TARGET_SIM
-                )
-            if _similarity_ratio(src_u, safe) > 0.92:
-                stats.reverted_source += 1
-            else:
-                stats.classical_kept += 1
-            by_para.setdefault(unit.paragraph_index, []).append(safe)
-        # Assembly invariant: every aligned source unit produced an output
-        paras_out = [
-            " ".join(by_para[i]).strip()
-            for i in sorted(by_para.keys())
-            if by_para[i]
-        ]
-        text = tidy(correct_text(_light_post_gen("\n\n".join(p for p in paras_out if p), tone)))
-        return text, stats, False
-
-    # --- Primary path: restructure first, then Small LM → validate ---
-    source_units = iter_source_units(source)
-    if not source_units:
-        return source, stats, False
-
-    lm_budget = int(GENERATIVE_MAX_PARAGRAPHS)
-    unlimited = lm_budget <= 0
-    lm_used = 0
-
-    outputs: list[str] = []
-    for para_idx, src_u in source_units:
-        stats.units += 1
-        # ALWAYS emit exactly one string for this source unit (no drops)
-        chosen = src_u
-
-        # Reorder-first: skip LM when dep+template restructure succeeds
-        if strength >= 1 and len(src_u.split()) >= 4:
-            restructured = _try_restructure_unit(src_u)
-            if restructured:
-                stats.bump_reason("restructure")
-                stats.classical_kept += 1
-                outputs.append(_ensure_unit_terminal(restructured))
-                continue
-
-        if not _looks_needs_lm(src_u, strength):
-            stats.skipped += 1
-            # Tiny leftovers: still force visible rewrite at Normal+ when possible
-            if strength >= 1 and len(src_u.split()) >= 4:
-                chosen = _require_visible_rewrite(
-                    src_u, src_u, tone, rng, target_sim=_VISIBLE_TARGET_SIM
-                )
-            outputs.append(_ensure_unit_terminal(chosen))
-            continue
-
-        if not unlimited and lm_used >= lm_budget:
-            stats.skipped += 1
-            outputs.append(
-                _ensure_unit_terminal(
-                    _require_visible_rewrite(
-                        src_u,
-                        _light_revert_polish(src_u, tone, rng),
-                        tone,
-                        rng,
-                        target_sim=_VISIBLE_TARGET_SIM,
-                    )
-                )
-            )
-            continue
-
-        stats.regenerated += 1
-        lm_used += 1
-        mark_gen_used()
-        used_gen = True
-
-        gen_raw = rewrite_unit(
-            src_u,
-            tone=tone,
-            strength=max(1, strength),
-            min_meaning=0.80,
-            max_sim=_LM_SHIP_MAX_SIM,
-        )
-        gen_clean = tidy(correct_text(gen_raw) if gen_raw else "")
-        gen_clean = _clamp_to_source_shape(src_u, gen_clean)
-        ok, reasons = _accept_lm_unit(src_u, gen_clean, max_sim=_LM_SHIP_MAX_SIM)
-
-        if ok:
-            stats.gen_accepted += 1
-            chosen = gen_clean
-        else:
-            soft = not _hard_fail([r.split(":")[0] for r in reasons])
-            if soft or "near_copy" in reasons or "too_similar" in reasons:
-                gen_raw2 = rewrite_unit(
-                    src_u,
-                    tone=tone,
-                    strength=2,
-                    min_meaning=0.78,
-                    max_sim=0.85,
-                )
-                gen_clean2 = tidy(correct_text(gen_raw2) if gen_raw2 else "")
-                gen_clean2 = _clamp_to_source_shape(src_u, gen_clean2)
-                ok2, reasons2 = _accept_lm_unit(src_u, gen_clean2, max_sim=0.85)
-                if ok2:
-                    stats.gen_accepted += 1
-                    chosen = gen_clean2
-                else:
-                    for r in reasons2:
-                        stats.bump_reason(r)
-                    stats.reverted_source += 1
-                    logger.info(
-                        "LM unit rejected (%s); visible paraphrase on source unit",
-                        reasons2 or reasons or ["near_copy"],
-                    )
-                    chosen = _light_revert_polish(src_u, tone, rng, strength=1)
-            else:
-                for r in reasons:
-                    stats.bump_reason(r)
-                stats.reverted_source += 1
-                logger.info(
-                    "LM unit hard-rejected (%s); visible paraphrase on source unit",
-                    reasons,
-                )
-                chosen = _light_revert_polish(src_u, tone, rng, strength=1)
-
-        if not (chosen or "").strip():
-            chosen = src_u
-        chosen = _clamp_to_source_shape(src_u, chosen) or src_u
-        # Never let an accepted unit expand into multiple sentences
-        if _sentence_count_simple(chosen) != _sentence_count_simple(src_u):
-            if _sentence_count_simple(src_u) == 1:
-                chosen = _light_revert_polish(src_u, tone, rng, strength=1)
-            else:
-                chosen = src_u
-        # Must not ship grammar-only / near-copy — force visible rewrite
-        chosen = _require_visible_rewrite(
-            src_u, chosen, tone, rng, target_sim=_VISIBLE_TARGET_SIM
-        )
-        outputs.append(_ensure_unit_terminal(chosen))
-
-    if len(outputs) != len(source_units):
-        logger.warning(
-            "Assembly invariant broken (%s out vs %s in); repairing from source",
-            len(outputs),
-            len(source_units),
-        )
-        while len(outputs) < len(source_units):
-            outputs.append(_ensure_unit_terminal(source_units[len(outputs)][1]))
-        outputs = outputs[: len(source_units)]
-
-    # Per-slot sanity: empty/too-short slots → source unit then forced rewrite
-    fixed_outs: list[str] = []
-    for (_pi, src_u), out in zip(source_units, outputs):
-        piece = (out or "").strip()
-        if (
-            not piece
-            or _sentence_count_simple(piece) < _sentence_count_simple(src_u)
-            or len(piece.split()) < max(3, int(len(src_u.split()) * 0.55))
-        ):
-            piece = _require_visible_rewrite(
-                src_u, src_u, tone, rng, target_sim=_VISIBLE_TARGET_SIM
-            )
-            stats.bump_reason("unit_restore")
-        fixed_outs.append(_ensure_unit_terminal(piece))
-    outputs = fixed_outs
-
-    # Doc-level: any remaining near-copies get forced paraphrase
-    boosted_outs: list[str] = []
-    for (_pi, src_u), out in zip(source_units, outputs):
-        if (
-            strength >= 1
-            and len(src_u.split()) >= 4
-            and _similarity_ratio(src_u, out) > _VISIBLE_TARGET_SIM
-        ):
-            boosted_outs.append(
-                _require_visible_rewrite(
-                    src_u, out, tone, rng, target_sim=_VISIBLE_TARGET_SIM
-                )
-            )
-            stats.bump_reason("near_copy_boost")
-        else:
-            boosted_outs.append(out)
-    outputs = boosted_outs
-
-    # Assemble ONLY from per-unit strings — no document-level grammar here
-    text = _assemble_hybrid_units(source_units, outputs)
-    scrubbed = tidy(_light_post_gen(text, tone))
-    if len(iter_source_units(scrubbed)) < len(source_units):
-        logger.warning(
-            "Post-scrub dropped units (%s→%s); keeping assembled text",
-            len(source_units),
-            len(iter_source_units(scrubbed)),
-        )
-        text = _assemble_hybrid_units(source_units, outputs)
-    else:
-        text = scrubbed
-    # Final hard guarantee — never ship fewer units than source
-    if len(iter_source_units(text)) < len(source_units):
-        logger.warning("Final hybrid text lost units; restoring assembled outputs")
-        text = _assemble_hybrid_units(source_units, outputs)
-    if len(iter_source_units(text)) < len(source_units):
-        text = _assemble_hybrid_units(
-            source_units,
-            [_ensure_unit_terminal(u) for _i, u in source_units],
-        )
-    return text, stats, used_gen
+    sentences: list | None = None
+    mapping: list | None = None
+    stats: dict | None = None
 
 
 def rewrite_text(
@@ -1284,312 +61,84 @@ def rewrite_text(
     ml_polish: bool = False,
 ) -> RewriteResult:
     """
-    Quality-first rewrite.
+    Structural document rewrite (rule-based sentence reorder).
 
-    PIPELINE_MODE=hybrid (default with ML): grammar → Small LM per sentence → validate
-      (classical lexicon only if generative unavailable)
-    PIPELINE_MODE=generative: whole-document generative rewrite
-    PIPELINE_MODE=classical / ml_polish off: classical lexicon path
+    ``tone``, ``strength``, ``preserve_length``, and ``ml_polish`` are accepted
+    for API compatibility but do not drive synonym or generative rewrite.
+    MiniLM is used only when ENGINE_USE_MINILM_SAFETY is enabled.
     """
+    _ = preserve_length
     started = time.perf_counter()
-    original = normalize_whitespace(text or "")
-    if not original:
-        raise ValueError("Paste some text first.")
-
-    source = original
-    grammar_fixed_input = False
-    if GRAMMAR_FIX_INPUT:
-        cleaned = correct_text(original)
-        if cleaned and cleaned.strip():
-            grammar_fixed_input = cleaned.strip() != original.strip()
-            source = normalize_whitespace(cleaned)
-
-    strength = max(0, min(2, int(strength)))
     tone = normalize_tone(tone)
-    mode = PIPELINE_MODE if ml_polish else "classical"
-    if mode == "classical" and not ml_polish:
-        pass
-    elif not ml_polish:
-        mode = "classical"
-    elif mode == "hybrid" and not GENERATIVE_PRIMARY:
-        # hybrid without gen primary still runs classical+validate; patch only if model loads
-        pass
-
-    gen_ready = bool(ml_polish) and generative_available()
-    want_minilm = bool(ml_polish) and minilm_available()
-    # Skip MiniLM synonym ranking on generative-heavy paths
-    set_ml_polish(want_minilm and mode == "classical")
-    rng = _rng_for(
-        source + "|" + tone + "|" + str(strength) + ("|ml" if ml_polish else "")
-    )
-
-    notes = ""
-    used_gen_main = False
-    hybrid_stats: HybridStats | None = None
-    rewritten = source
-
-    if mode == "hybrid" and ml_polish:
-        logger.info(
-            "Hybrid path: LM-per-sentence → validate (%s, model=%s, backend=%s)…",
-            tone,
-            GENERATIVE_MODEL if gen_ready else "-",
-            GENERATIVE_BACKEND if gen_ready else "-",
-        )
-        rewritten, hybrid_stats, used_gen_main = _hybrid_rewrite(
-            source,
-            tone,
-            strength,
-            rng,
-            can_generate=gen_ready,
-        )
-        if not gen_ready:
-            tip = (
-                "Hybrid: generative unavailable; classical lexicon fallback only."
-            )
-            notes = tip
-        else:
-            tip = (
-                f"Hybrid LM: accepted={hybrid_stats.gen_accepted}/"
-                f"{hybrid_stats.regenerated} "
-                f"skipped={hybrid_stats.skipped} "
-                f"reverted={hybrid_stats.reverted_source} "
-                f"classical_fb={hybrid_stats.classical_kept}"
-            )
-            notes = tip
-
-    elif mode == "generative" and ml_polish and gen_ready:
-        logger.info(
-            "Generative path (%s, model=%s, backend=%s)…",
-            tone,
-            GENERATIVE_MODEL,
-            GENERATIVE_BACKEND,
-        )
-        draft = generative_paraphrase(
-            source,
-            tone=tone,
-            strength=max(1, strength),
-            min_meaning=0.80,
-            max_sim=0.92,
-        )
-        if draft and draft.strip() and _accept_generative(source, draft):
-            sim0 = _similarity_ratio(source, draft)
-            if sim0 >= 0.92:
-                logger.info(
-                    "Generative draft too similar (%.3f); trying stronger pass",
-                    sim0,
-                )
-                draft2 = generative_paraphrase(
-                    source,
-                    tone=tone,
-                    strength=2,
-                    min_meaning=0.78,
-                    max_sim=0.90,
-                )
-                if (
-                    draft2
-                    and draft2.strip()
-                    and _accept_generative(source, draft2)
-                    and _similarity_ratio(source, draft2) < 0.92
-                ):
-                    mark_gen_used()
-                    used_gen_main = True
-                    rewritten = _light_post_gen(tidy(draft2), tone)
-                else:
-                    used_gen_main = False
-                    tip = (
-                        "Generative rewrite stayed a near-copy; "
-                        "using classical fallback for visible change."
-                    )
-                    notes = tip
-            else:
-                mark_gen_used()
-                used_gen_main = True
-                rewritten = _light_post_gen(tidy(draft), tone)
-                if _similarity_ratio(source, rewritten) > 0.90:
-                    draft2 = generative_paraphrase(
-                        source,
-                        tone=tone,
-                        strength=2,
-                        min_meaning=0.78,
-                        max_sim=0.90,
-                    )
-                    if (
-                        draft2
-                        and draft2.strip()
-                        and _accept_generative(source, draft2)
-                        and _similarity_ratio(source, draft2) < 0.92
-                    ):
-                        rewritten = _light_post_gen(tidy(draft2), tone)
-                logger.info(
-                    "Generative main path words %s→%s sim=%.3f",
-                    word_count(source),
-                    word_count(rewritten),
-                    _similarity_ratio(source, rewritten),
-                )
-        else:
-            tip = (
-                "Generative rewrite unavailable or failed validation; "
-                "using classical fallback."
-                if draft
-                else "Generative model produced no draft; using classical fallback."
-            )
-            notes = tip
-            used_gen_main = False
-
-    if mode != "hybrid" and not used_gen_main:
-        if want_minilm:
-            set_ml_polish(True)
-        if LEXICON_FALLBACK or not ml_polish:
-            rewritten = _classical_rewrite(source, tone, strength, rng)
-        else:
-            rewritten = source
-            tip = "No generative output and lexicon fallback disabled; returning grammar-corrected source."
-            notes = f"{notes} {tip}".strip() if notes else tip
-
-        if want_minilm:
-            set_ml_polish(True)
-            rewritten = apply_minilm_polish(source, rewritten, tone)
-            rewritten = tidy(rewritten)
-            rewritten = apply_tone_contractions(rewritten, tone)
-
-        ratio_tmp = _similarity_ratio(source, rewritten)
-        if ratio_tmp > 0.88 and LEXICON_FALLBACK:
-            logger.info("Classical near-copy (%.3f); stronger lexicon pass", ratio_tmp)
-            rewritten = _classical_rewrite(source, tone, strength, rng, force=True)
-            if want_minilm:
-                rewritten = apply_minilm_polish(source, rewritten, tone)
-                rewritten = apply_tone_contractions(rewritten, tone)
-            tip = "Applied stronger classical pass (first pass too similar)."
-            notes = f"{notes} {tip}".strip() if notes else tip
-
-    rewritten = enforce_length_budget(source, rewritten, preserve_length)
-    rewritten = tidy(rewritten)
-
-    if used_gen_main and mode == "generative" and not _accept_generative(source, rewritten):
-        logger.info("Final generative output failed validation; reverting to corrected source")
-        rewritten = source
-        tip = "Generative output failed final meaning checks; returned grammar-corrected source."
-        notes = f"{notes} {tip}".strip() if notes else tip
-        used_gen_main = False
-
-    if GRAMMAR_FIX_OUTPUT:
-        cleaned_out = correct_text(rewritten)
-        if cleaned_out and cleaned_out.strip():
-            # Hybrid: never let document-level LT drop sentence units
-            if mode == "hybrid" and hybrid_stats and hybrid_stats.units:
-                n_src = hybrid_stats.units
-                if len(iter_source_units(cleaned_out)) < n_src:
-                    logger.warning(
-                        "Output grammar dropped units (%s→%s); keeping pre-grammar text",
-                        n_src,
-                        len(iter_source_units(cleaned_out)),
-                    )
-                else:
-                    rewritten = tidy(cleaned_out)
-            else:
-                rewritten = tidy(cleaned_out)
-
-    # Nuclear hybrid safety: never return fewer sentence units than the source
-    if mode == "hybrid" and hybrid_stats and hybrid_stats.units:
-        n_src = hybrid_stats.units
-        if len(iter_source_units(rewritten)) < n_src:
-            logger.warning(
-                "Post-pipeline unit loss (%s→%s); restoring from grammar source units",
-                n_src,
-                len(iter_source_units(rewritten)),
-            )
-            src_units = iter_source_units(source)
-            rewritten = _assemble_hybrid_units(
-                src_units,
-                [
-                    _ensure_unit_terminal(
-                        _light_revert_polish(u, tone, rng, strength=1)
-                    )
-                    for _i, u in src_units
-                ],
-            )
-
-    ratio = _similarity_ratio(source, rewritten)
-    user_sim = _similarity_ratio(original, rewritten)
-    changed = rewritten.strip() != original.strip()
-
-    engine_bits: list[str] = []
-    if GRAMMAR_FIX_INPUT or GRAMMAR_FIX_OUTPUT:
-        engine_bits.append("grammar")
-    if mode == "hybrid":
-        if used_gen_main or (hybrid_stats and hybrid_stats.regenerated):
-            engine_bits.append(f"lm-unit:{backend_kind()}")
-            engine_bits.extend(["validate", "rank"])
-        elif hybrid_stats and hybrid_stats.classical_kept:
-            engine_bits.extend(["rules", "structure", "lexicon-fallback"])
-        engine_bits.append("spacy" if spacy_available() else "regex-fallback")
-        engine_bits.append("scrub-light")
-    elif gen_was_used():
-        engine_bits.append(f"generative:{backend_kind()}")
-        engine_bits.append("validate")
-        engine_bits.append("rank")
-        engine_bits.append("spacy" if spacy_available() else "regex-fallback")
-        engine_bits.append("scrub-light")
-    else:
-        engine_bits.append("spacy" if spacy_available() else "regex-fallback")
-        engine_bits.extend(["rules", "structure", "lexicon-fallback", "mechanics", "tone"])
-    if want_minilm and ml_was_used():
-        engine_bits.append("minilm")
+    strength = max(0, min(2, int(strength)))
 
-    if ml_polish and not gen_ready and not want_minilm:
-        tip = (
-            "ML polish requested but generative/MiniLM models are unavailable "
-            "(install torch+transformers and/or fastembed; set GENERATIVE_MODEL)."
-        )
-        notes = f"{notes} {tip}".strip() if notes else tip
-    elif ml_polish and want_minilm and not gen_ready and mode != "hybrid":
-        tip = "ML polish: MiniLM guard only (generative unavailable — check GENERATIVE_MODEL)."
-        notes = f"{notes} {tip}".strip() if notes else tip
-    if grammar_fixed_input:
-        tip = "Corrected grammar in the source before rewriting."
-        notes = f"{notes} {tip}".strip() if notes else tip
+    result = rewrite_document(
+        text,
+        use_minilm_safety=bool(ml_polish) or None,
+    )
+
+    hybrid = HybridStats(
+        units=result.stats.sentences,
+        classical_kept=result.stats.rewritten,
+        regenerated=0,
+        gen_accepted=0,
+        reverted_source=result.stats.reverted,
+        skipped=result.stats.skipped,
+        reasons=dict(result.stats.reasons),
+    )
+
+    seconds = result.stats.seconds or round(time.perf_counter() - started, 3)
+    sentence_payload = [
+        {
+            "index": s.index,
+            "original": s.original,
+            "rewritten": s.rewritten,
+            "confidence": s.confidence,
+            "status": s.status,
+            "template_id": s.template_id,
+            "sentence_type": s.sentence_type,
+            "reasons": s.reasons,
+        }
+        for s in result.sentences
+    ]
+    stats_payload = {
+        "batches": result.stats.batches,
+        "blocks": result.stats.blocks,
+        "sentences": result.stats.sentences,
+        "rewritten": result.stats.rewritten,
+        "skipped": result.stats.skipped,
+        "reverted": result.stats.reverted,
+        "passthrough": result.stats.passthrough,
+        "seconds": result.stats.seconds,
+        "reasons": result.stats.reasons,
+    }
 
-    engine = "+".join(engine_bits)
-    msg = (
-        f"rewrite engine={engine} mode={mode} strength={strength} tone={tone} "
-        f"words={word_count(original)}->{word_count(rewritten)} "
-        f"changed={changed} similarity={ratio:.3f} "
-        f"ml_polish={bool(ml_polish)} gen={gen_ready} gen_main={used_gen_main} "
-        f"backend={GENERATIVE_BACKEND if gen_ready else '-'} "
-        f"model={GENERATIVE_MODEL if gen_ready else '-'}"
+    logger.info(
+        "rewrite_text structural words=%s→%s changed=%s t=%.2fs",
+        result.input_words,
+        result.output_words,
+        result.changed,
+        seconds,
     )
-    logger.info(msg)
-    print(f"[plainrewrite] {msg}", flush=True)
-
-    if not changed:
-        tip = (
-            "Output matched input after validation. "
-            "Try Heavy strength, enable ML polish with a loaded generative model, "
-            "or paste text with more rewritable content."
-        )
-        notes = f"{notes} {tip}".strip() if notes else tip
-    elif used_gen_main and ratio > 0.90:
-        tip = "Rewrite stayed close to the source (meaning-first ranking)."
-        notes = f"{notes} {tip}".strip() if notes else tip
-    if user_sim >= 0.92 and changed:
-        tip = f"Surface similarity to input is high ({user_sim:.0%})."
-        notes = f"{notes} {tip}".strip() if notes else tip
 
     return RewriteResult(
-        text=rewritten,
-        input_words=word_count(original),
-        output_words=word_count(rewritten),
-        seconds=round(time.perf_counter() - started, 3),
-        engine=engine,
+        text=result.text,
+        input_words=result.input_words or word_count(text or ""),
+        output_words=result.output_words or word_count(result.text),
+        seconds=seconds,
+        engine=result.engine,
         strength=strength,
         tone=tone,
-        changed=changed,
-        notes=notes,
-        pipeline_mode=mode,
-        hybrid=hybrid_stats,
-        similarity=round(user_sim, 3),
+        changed=result.changed,
+        notes=result.notes,
+        pipeline_mode="structural",
+        hybrid=hybrid,
+        similarity=result.similarity,
+        sentences=sentence_payload,
+        mapping=result.mapping,
+        stats=stats_payload,
     )
 
 
 def similarity_check(rewrite: str, reference: str) -> SimilarityResult:
-    return compare_similarity(rewrite or "", reference or "")
+    return compare_similarity(rewrite, reference)

app/pipeline/restructure.py

@@ -1,425 +1,88 @@
-"""Dependency + template sentence restructure (reorder-first, no synonym thrash).
+"""Dependency + template sentence restructure — thin re-export of engine stages.
 
-Pipeline per sentence:
-  classify → parse slots → plan → template → fill → return candidate
-  (caller runs grammar repair + semantic safety)
+Prefer ``app.engine`` for new code. Kept for scripts that import restructure APIs.
 """
 
 from __future__ import annotations
 
-import re
-from dataclasses import dataclass, field
-
-from app.pipeline.nlp import get_nlp
-
-
-_TIME_WORDS = frozenset(
-    """
-    yesterday today tomorrow earlier later recently now tonight
-    monday tuesday wednesday thursday friday saturday sunday
-    morning afternoon evening night
-    """.split()
-)
-
-_PLACE_PREPS = frozenset("to at in on into onto from toward towards".split())
-_MOTION_VERBS = frozenset(
-    """
-    went go goes going gone walk walks walked walking
-    run runs ran running drive drives drove driving
-    come comes came coming move moves moved moving
-    fly flies flew flying ride rides rode riding
-    travel travels traveled travelled travel
-    head heads headed heading return returns returned
-    """.split()
+from app.engine.classify import classify_sentence
+from app.engine.models import RewritePlan as ReorderPlan
+from app.engine.models import SentenceSlots
+from app.engine.parse import extract_slots
+from app.engine.plan import build_plan
+from app.engine.rewrite import generate_from_plan, reorder_quality_ok
+from app.engine.templates import (
+    fill_template as _fill_template,
+    rank_templates,
+    try_because_front,
+    try_discourse_front,
 )
 
 
-@dataclass
-class SentenceSlots:
-    """Constituent spans extracted for template fill."""
-
-    text: str
-    subject: str = ""
-    verb: str = ""
-    verb_phrase: str = ""  # auxiliaries + main verb
-    object: str = ""
-    place: str = ""
-    time: str = ""
-    manner: str = ""
-    leftover: str = ""
-    confidence: float = 0.0
-    sentence_type: str = "unsupported"
-    reasons: list[str] = field(default_factory=list)
-
-
-@dataclass
-class ReorderPlan:
-    template_id: str
-    slots: SentenceSlots
-    confidence: float
-
-
-@dataclass
 class RestructureResult:
-    text: str
-    template_id: str
-    confidence: float
-    plan: ReorderPlan | None = None
-
-
-def classify_sentence(text: str) -> str:
-    """Tag sentence type; only safe types are restructured."""
-    t = (text or "").strip()
-    if not t:
-        return "empty"
-    if re.match(r"^#{1,6}\s", t) or (len(t.split()) <= 6 and t.isupper()):
-        return "title"
-    if re.match(r"^(\d+[\.\)]\s+|[-*•]\s+)", t):
-        return "list_item"
-    if t.count('"') >= 2 or t.count("“") or t.count("'") >= 2:
-        # Allow short quotes inside; skip heavily quoted lines
-        if t.startswith(("\"", "'", "“", "‘")):
-            return "quoted"
-    if "?" in t or t.endswith("?"):
-        return "question"
-    if len(t.split()) < 3:
-        return "too_short"
-    if len(t.split()) > 45:
-        return "too_long"
-    return "simple_declarative"
-
-
-def _span_text(tokens: list) -> str:
-    if not tokens:
-        return ""
-    return "".join(t.text_with_ws for t in tokens).strip()
-
-
-def _subtree_tokens(token) -> list:
-    return sorted(token.subtree, key=lambda t: t.i)
+    def __init__(
+        self,
+        text: str,
+        template_id: str,
+        confidence: float,
+        plan: ReorderPlan | None = None,
+    ) -> None:
+        self.text = text
+        self.template_id = template_id
+        self.confidence = confidence
+        self.plan = plan
 
 
-def extract_slots(text: str) -> SentenceSlots:
-    """Parse subject/verb/object/time/place/manner with spaCy when available."""
-    raw = (text or "").strip()
-    slots = SentenceSlots(text=raw, sentence_type=classify_sentence(raw))
-    if slots.sentence_type not in {"simple_declarative", "compound"}:
-        slots.reasons.append(f"skip:{slots.sentence_type}")
-        return slots
-
-    nlp = get_nlp()
-    if nlp is None:
-        return _extract_slots_regex(raw)
-
-    doc = nlp(raw)
-    # Prefer ROOT verb
-    root = next((t for t in doc if t.dep_ == "ROOT" and t.pos_ in {"VERB", "AUX"}), None)
-    if root is None:
-        root = next((t for t in doc if t.pos_ == "VERB"), None)
-    if root is None:
-        slots.reasons.append("no_verb")
-        return slots
-
-    # Subject
-    subj_toks: list = []
-    for child in root.children:
-        if child.dep_ in {"nsubj", "nsubjpass"}:
-            subj_toks = _subtree_tokens(child)
-            break
-    slots.subject = _span_text(subj_toks)
-
-    # Verb phrase: aux + root (exclude negation particles for separate handling later)
-    verb_toks = [t for t in root.lefts if t.dep_ in {"aux", "auxpass"}] + [root]
-    slots.verb = root.text
-    slots.verb_phrase = _span_text(sorted(verb_toks, key=lambda t: t.i))
-
-    # Object / attr / dobj
-    obj_toks: list = []
-    for child in root.children:
-        if child.dep_ in {"dobj", "obj", "attr", "xcomp", "ccomp"}:
-            # Prefer noun-ish objects over full clause for simple templates
-            if child.dep_ in {"ccomp", "xcomp"} and child.pos_ == "VERB":
-                continue
-            obj_toks = _subtree_tokens(child)
-            break
-    # Strip trailing advmod/time from object subtree if present
-    slots.object = _span_text(obj_toks)
-
-    # Place: prep "to/at/in..." with pobj
-    place_parts: list[str] = []
-    for child in root.children:
-        if child.dep_ == "prep" and child.text.lower() in _PLACE_PREPS:
-            place_parts.append(_span_text(_subtree_tokens(child)))
-        elif child.dep_ == "prt":
-            pass
-    # Also "went to school" — prep attached to verb
-    if not place_parts:
-        for t in doc:
-            if t.dep_ == "prep" and t.head == root and t.text.lower() in _PLACE_PREPS:
-                place_parts.append(_span_text(_subtree_tokens(t)))
-    # Motion verb + "to X" misparsed as infinitive xcomp (e.g. drove to work)
-    if not place_parts and root.text.lower() in _MOTION_VERBS:
-        for child in root.children:
-            if child.dep_ == "xcomp":
-                aux_to = [
-                    t
-                    for t in child.lefts
-                    if t.dep_ == "aux" and t.text.lower() == "to"
-                ]
-                if aux_to or child.text.lower() not in {"be", "have", "do"}:
-                    place_parts.append("to " + child.text)
-                    break
-    slots.place = " ".join(p for p in place_parts if p).strip()
-
-    # Time + manner from advmod / npadvmod / prep
-    time_parts: list[str] = []
-    manner_parts: list[str] = []
-    for t in doc:
-        low = t.text.lower()
-        if t.dep_ in {"advmod", "npadvmod"} and (t.head == root or t.head.head == root):
-            if low in _TIME_WORDS or t.ent_type_ == "DATE" or t.ent_type_ == "TIME":
-                time_parts.append(_span_text(_subtree_tokens(t)))
-            elif low.endswith("ly") or t.pos_ == "ADV":
-                # avoid degree modifiers alone
-                if low not in {"very", "really", "quite", "just", "also", "still", "not", "n't"}:
-                    manner_parts.append(t.text)
-        if t.dep_ == "npadvmod" and low in _TIME_WORDS:
-            time_parts.append(t.text)
-        if t.ent_type_ in {"DATE", "TIME"} and t.head == root:
-            time_parts.append(t.text)
-
-    # Fallback scan for known time words anywhere
-    if not time_parts:
-        for t in doc:
-            if t.text.lower() in _TIME_WORDS:
-                time_parts.append(t.text)
-    if not manner_parts:
-        for t in doc:
-            if t.pos_ == "ADV" and t.text.lower().endswith("ly"):
-                manner_parts.append(t.text)
-
-    slots.time = " ".join(dict.fromkeys(time_parts)).strip()
-    slots.manner = " ".join(dict.fromkeys(manner_parts)).strip()
-
-    # Clean object if it absorbed place/time/manner
-    for piece in (slots.place, slots.time, slots.manner):
-        if piece and slots.object.endswith(piece):
-            slots.object = slots.object[: -len(piece)].strip(" ,")
-
-    # Confidence
-    score = 0.2
-    if slots.subject:
-        score += 0.25
-    if slots.verb_phrase:
-        score += 0.2
-    if slots.place or slots.object:
-        score += 0.15
-    if slots.time:
-        score += 0.1
-    if slots.manner:
-        score += 0.1
-    slots.confidence = min(1.0, score)
-    slots.sentence_type = "simple_declarative"
-    if slots.confidence < 0.45 or not slots.subject or not slots.verb_phrase:
-        slots.reasons.append("low_confidence")
-    return slots
-
-
-def _extract_slots_regex(text: str) -> SentenceSlots:
-    """Regex fallback when spaCy is unavailable (covers Ram-style sentences)."""
-    slots = SentenceSlots(text=text, sentence_type=classify_sentence(text))
-    if slots.sentence_type != "simple_declarative":
-        return slots
-    end = ""
-    core = text
-    m_end = re.search(r"[.!?]+$", text)
-    if m_end:
-        end = m_end.group(0)
-        core = text[: m_end.start()]
-
-    m = re.match(
-        r"^(?P<subj>[A-Za-z][\w'-]*)\s+"
-        r"(?P<verb>went|go|goes|walked|ran|drove|came|moved|is|are|was|were|has|have|had)\s+"
-        r"(?:to\s+(?P<place>.+?)\s+)?"
-        r"(?P<time>yesterday|today|tomorrow|earlier|later|recently)?\s*"
-        r"(?P<manner>\w+ly)?$",
-        core,
-        flags=re.I,
+def plan_reorder(slots: SentenceSlots) -> ReorderPlan | None:
+    cands = rank_templates(slots)
+    if not cands:
+        return None
+    return ReorderPlan(
+        safe=True,
+        slots=slots,
+        template_id=cands[0].template_id,
+        candidates=cands,
+        confidence=cands[0].confidence,
     )
-    if not m:
-        # Broader: Subj verb ... time manner
-        m2 = re.match(
-            r"^(?P<subj>[A-Za-z][\w'-]*)\s+(?P<rest>.+)$",
-            core,
-            flags=re.I,
-        )
-        if not m2:
-            slots.reasons.append("regex_no_match")
-            return slots
-        slots.subject = m2.group("subj")
-        rest = m2.group("rest")
-        # peel manner / time from end
-        mm = re.search(r"\b(\w+ly)$", rest, flags=re.I)
-        if mm:
-            slots.manner = mm.group(1)
-            rest = rest[: mm.start()].strip()
-        mt = re.search(
-            r"\b(yesterday|today|tomorrow|earlier|later|recently)$",
-            rest,
-            flags=re.I,
-        )
-        if mt:
-            slots.time = mt.group(1)
-            rest = rest[: mt.start()].strip()
-        mv = re.match(r"^(\w+)(?:\s+to\s+(.+))?$", rest, flags=re.I)
-        if mv:
-            slots.verb_phrase = mv.group(1)
-            slots.verb = mv.group(1)
-            slots.place = f"to {mv.group(2)}" if mv.group(2) else ""
-            if slots.place and not slots.place.lower().startswith("to "):
-                slots.place = "to " + slots.place
-        slots.confidence = 0.55 if slots.subject and slots.verb_phrase else 0.2
-        return slots
-
-    slots.subject = m.group("subj")
-    slots.verb = m.group("verb")
-    slots.verb_phrase = m.group("verb")
-    if m.group("place"):
-        slots.place = "to " + m.group("place").strip()
-    slots.time = (m.group("time") or "").strip()
-    slots.manner = (m.group("manner") or "").strip()
-    slots.confidence = 0.7
-    _ = end
-    return slots
-
-
-def _cap(text: str) -> str:
-    t = (text or "").strip()
-    if not t:
-        return t
-    return t[0].upper() + t[1:]
 
 
-def _join_slots(*parts: str) -> str:
-    bits = [p.strip() for p in parts if p and p.strip()]
-    s = " ".join(bits)
-    s = re.sub(r"\s+", " ", s).strip()
-    s = re.sub(r"\s+([,.;:!?])", r"\1", s)
-    return s
-
-
-def plan_reorder(slots: SentenceSlots) -> ReorderPlan | None:
-    """Pick a deterministic template from available slots."""
-    if slots.confidence < 0.45 or "low_confidence" in slots.reasons:
+def fill_template(plan: ReorderPlan):
+    """Back-compat: accept ReorderPlan like the old API."""
+    if plan is None or not plan.slots:
         return None
-    if not slots.subject or not slots.verb_phrase:
+    tid = plan.template_id or (plan.candidates[0].template_id if plan.candidates else "")
+    if not tid:
         return None
-
-    # Family A: has time + manner + place  → Time, Subject Manner Verb Place
-    if slots.time and slots.manner and slots.place:
-        return ReorderPlan("time_subj_manner_verb_place", slots, 0.9)
-
-    # Family B: time + place
-    if slots.time and slots.place:
-        return ReorderPlan("time_subj_verb_place", slots, 0.8)
-
-    # Family C: time + object
-    if slots.time and slots.object:
-        return ReorderPlan("time_subj_verb_object", slots, 0.75)
-
-    # Family D: manner + place
-    if slots.manner and slots.place:
-        return ReorderPlan("subj_manner_verb_place", slots, 0.7)
-
-    # Family E: time only → front time
-    if slots.time:
-        return ReorderPlan("time_front", slots, 0.65)
-
-    # Family F: manner only → manner before verb
-    if slots.manner:
-        return ReorderPlan("subj_manner_verb_rest", slots, 0.6)
-
-    # Family G: place present → keep SVP but ensure clean order
-    if slots.place:
-        return ReorderPlan("subj_verb_place", slots, 0.55)
-
-    if slots.object:
-        return ReorderPlan("subj_verb_object", slots, 0.5)
-
-    return None
-
-
-def fill_template(plan: ReorderPlan) -> str | None:
-    """Build sentence from plan without synonym replacement."""
-    s = plan.slots
-    end = "."
-    if s.text.rstrip().endswith(("!", "?")):
-        end = s.text.rstrip()[-1]
-
-    tid = plan.template_id
-    if tid == "time_subj_manner_verb_place":
-        # Yesterday, Ram happily went to school.
-        place = s.place
-        body = _join_slots(_cap(s.time) + ",", s.subject, s.manner, s.verb_phrase, place)
-        return body + end
-
-    if tid == "time_subj_verb_place":
-        body = _join_slots(_cap(s.time) + ",", s.subject, s.verb_phrase, s.place)
-        return body + end
-
-    if tid == "time_subj_verb_object":
-        body = _join_slots(_cap(s.time) + ",", s.subject, s.verb_phrase, s.object)
-        return body + end
-
-    if tid == "subj_manner_verb_place":
-        body = _join_slots(_cap(s.subject), s.manner, s.verb_phrase, s.place)
-        return body + end
-
-    if tid == "time_front":
-        # Move time to front; rebuild remainder by stripping time from original lightly
-        rest = s.text
-        if s.time:
-            rest = re.sub(rf"\b{re.escape(s.time)}\b", "", rest, count=1, flags=re.I)
-            rest = re.sub(r"\s+", " ", rest).strip(" ,.")
-        body = _join_slots(_cap(s.time) + ",", rest[0].lower() + rest[1:] if rest else "")
-        return (body.rstrip(".!?") + end) if body else None
-
-    if tid == "subj_manner_verb_rest":
-        # Subject manner verb ... (strip trailing manner from original rebuild)
-        rest_bits = [s.verb_phrase, s.object, s.place, s.time]
-        body = _join_slots(_cap(s.subject), s.manner, *rest_bits)
-        return body + end
-
-    if tid == "subj_verb_place":
-        body = _join_slots(_cap(s.subject), s.verb_phrase, s.place, s.time, s.manner)
-        return body + end
-
-    if tid == "subj_verb_object":
-        body = _join_slots(_cap(s.subject), s.verb_phrase, s.object, s.place, s.time, s.manner)
-        return body + end
-
-    return None
+    return _fill_template(tid, plan.slots)
 
 
 def restructure_sentence(text: str, *, min_confidence: float = 0.55) -> RestructureResult | None:
-    """Run classify → slots → plan → template fill. Returns None if unsafe/low-confidence."""
-    raw = (text or "").strip()
-    if not raw:
+    plan = build_plan(text, min_confidence=min_confidence)
+    if not plan.safe:
         return None
-    slots = extract_slots(raw)
-    if slots.confidence < min_confidence or slots.reasons:
-        # Allow regex path without explicit low_confidence reason if score ok
-        if "low_confidence" in slots.reasons or slots.confidence < min_confidence:
-            return None
-    plan = plan_reorder(slots)
-    if plan is None or plan.confidence < min_confidence:
-        return None
-    filled = fill_template(plan)
-    if not filled or filled.strip().lower().rstrip(".!?") == raw.lower().rstrip(".!?"):
+    filled = generate_from_plan(plan)
+    if not filled:
         return None
     return RestructureResult(
-        text=filled.strip(),
+        text=filled,
         template_id=plan.template_id,
         confidence=plan.confidence,
         plan=plan,
     )
+
+
+_reorder_quality_ok = reorder_quality_ok
+
+__all__ = [
+    "SentenceSlots",
+    "ReorderPlan",
+    "RestructureResult",
+    "classify_sentence",
+    "extract_slots",
+    "plan_reorder",
+    "fill_template",
+    "restructure_sentence",
+    "try_because_front",
+    "try_discourse_front",
+    "_reorder_quality_ok",
+]

app/pipeline/synonym.py

@@ -1,616 +1,75 @@
-"""Word-level synonym / register rewrite — dynamic per token (no sentence templates)."""
+"""Legacy synonym / register rewrite — NOT used by the structural engine.
+
+Kept only for ``load_protected_phrases`` (safety validator) and optional
+ALLOW_SYNONYM_REPLACEMENT experiments. Swap JSON maps have been removed.
+"""
 
 from __future__ import annotations
 
 import json
-import random
-import re
-from difflib import SequenceMatcher
 from functools import lru_cache
 
-from app.config import DATA_DIR
-from app.pipeline.ml_context import mark_ml_used, ml_polish_enabled
-from app.pipeline.minilm import pick_best_candidate
-from app.pipeline.nlp import get_nlp
-from app.pipeline.tones import is_academic, is_casual, is_elevated, normalize_tone
-
-_WORDNET_READY = False
-
-_STOP_SWAP = {
-    "be", "is", "are", "was", "were", "been", "being", "have", "has", "had",
-    "do", "does", "did", "will", "would", "could", "should", "may", "might",
-    "must", "shall", "can", "to", "of", "in", "for", "on", "with", "at", "by",
-    "from", "as", "into", "the", "a", "an", "and", "or", "but", "if",
-    "this", "that", "these", "those", "it", "its", "they", "them", "he", "she",
-    "we", "you", "i", "not", "no", "yes", "my", "our", "your", "their", "his",
-    "her", "who", "what", "when", "where", "why", "how", "which", "than", "then",
-    "nor", "yet", "both", "each", "other", "some", "such", "own", "same", "too",
-    # Prepositions / conjunctions — WordNet turns these into adverbs and breaks grammar
-    "after", "before", "during", "until", "since", "while", "among", "between",
-    "under", "over", "about", "against", "upon", "onto", "into", "through",
-    "across", "around", "without", "within", "beside", "behind", "above", "below",
-    # Degree adverbs — WordNet turns very→rattling and wrecks collocations
-    "very", "really", "quite", "rather", "pretty", "fairly", "highly", "extremely",
-    # Temporal discourse — WordNet nowadays→today is pointless churn
-    "nowadays", "today", "tonight", "tomorrow", "yesterday", "now",
-    # Collocation anchors — WordNet adj sense of "chance"→casual/nonchalant wrecks ESL text
-    "chance", "chances", "instead", "prefer", "prefers", "preferred", "less", "more",
-    "most", "least", "body", "fit", "health", "healthy", "unhealthy",
-    "unfortunately", "fortunately", "however", "therefore",
-}
-
-# Closed-class grammar only — no domain word lists.
-# Dynamic safety: POS gating (adj/adv WordNet only) + structural filters below.
-
-_OPENERS = {
-    # Conversational — short spoken connectors
-    "Casual": {
-        "but": "Still",
-        "so": "So",
-        "and": "Plus",
-        "yet": "Still",
-        "however": "Still",
-        "therefore": "So",
-        "additionally": "Also",
-        "furthermore": "Also",
-        "moreover": "Also",
-    },
-    # Business / professional — restrained connectors
-    "Formal": {
-        "but": "Yet",
-        "so": "Thus",
-        "and": "Also",
-        "yet": "Still",
-        "also": "Additionally",
-        "therefore": "Accordingly",
-        "however": "Nevertheless",
-    },
-    # Scholarly — denser discourse markers
-    "Academic": {
-        "but": "However",
-        "so": "Consequently",
-        "and": "Furthermore",
-        "yet": "Nevertheless",
-        "still": "Nevertheless",
-        "also": "Moreover",
-        "therefore": "Accordingly",
-        "thus": "Hence",
-    },
-}
-
-_IRREGULAR_PAST = {
-    "heard", "buried", "wished", "decided", "stretched", "pushed", "stopped",
-    "tried", "kept", "made", "said", "went", "came", "took", "gave", "found",
-    "thought", "told", "knew", "began", "grew", "saw", "sat", "stood", "ran",
-    "felt", "laid", "lain", "sung", "sang", "chosen", "chose", "done", "been",
-    "had", "did", "was", "were", "got", "put", "let", "set", "cut", "hit",
-    "hurt", "read", "led", "met", "paid", "spent", "built", "meant", "left",
-    "brought", "bought", "caught", "taught", "fought", "sought",
-}
-
-
-def _ensure_wordnet() -> bool:
-    global _WORDNET_READY
-    if _WORDNET_READY:
-        return True
-    try:
-        from nltk.corpus import wordnet as wn
-
-        try:
-            wn.synsets("test")
-        except LookupError:
-            import nltk
-
-            nltk.download("wordnet", quiet=True)
-            nltk.download("omw-1.4", quiet=True)
-        _WORDNET_READY = True
-        return True
-    except Exception:
-        return False
+from app.config import ALLOW_SYNONYM_REPLACEMENT, DATA_DIR
 
 
 def _load_json_map(name: str) -> dict[str, str]:
     path = DATA_DIR / name
-    if not path.exists():
+    if not path.is_file():
+        return {}
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+        return {str(k): str(v) for k, v in data.items()} if isinstance(data, dict) else {}
+    except Exception:
         return {}
-    return {k.lower(): v for k, v in json.loads(path.read_text(encoding="utf-8")).items() if k != v}
 
 
 @lru_cache(maxsize=1)
 def load_protected_phrases() -> tuple[str, ...]:
-    """Multiword collocations whose component words must not be swapped.
-
-    Data-driven (app/data/protected_phrases.json) so it stays domain-agnostic —
-    e.g. 'fast food' must never become 'rapid food'.
-    """
     path = DATA_DIR / "protected_phrases.json"
-    if not path.exists():
+    if not path.is_file():
         return ()
     try:
-        raw = json.loads(path.read_text(encoding="utf-8"))
+        data = json.loads(path.read_text(encoding="utf-8"))
+        if isinstance(data, list):
+            phrases = {str(p).strip().lower() for p in data if str(p).strip()}
+        elif isinstance(data, dict):
+            phrases = {str(x).strip().lower() for x in data.keys() if str(x).strip()}
+        else:
+            return ()
+        return tuple(sorted(phrases, key=len, reverse=True))
     except Exception:
         return ()
-    phrases = {str(p).strip().lower() for p in raw if str(p).strip()}
-    # Longest first so multi-word matches win.
-    return tuple(sorted(phrases, key=len, reverse=True))
 
 
-def _protected_spans(sentence: str) -> list[tuple[int, int]]:
-    """Character spans in `sentence` covered by a protected collocation."""
-    phrases = load_protected_phrases()
-    if not phrases:
-        return []
-    spans: list[tuple[int, int]] = []
-    low = sentence.lower()
-    for phrase in phrases:
-        for m in re.finditer(rf"\b{re.escape(phrase)}\b", low):
-            spans.append((m.start(), m.end()))
-    return spans
-
-
-def _in_protected_span(start: int, end: int, spans: list[tuple[int, int]]) -> bool:
-    for s, e in spans:
-        if start < e and end > s:  # any overlap
-            return True
-    return False
-
-
-@lru_cache(maxsize=1)
 def load_preferred_swaps() -> dict[str, str]:
     return _load_json_map("preferred_swaps.json")
 
 
-@lru_cache(maxsize=1)
 def load_elevate_swaps() -> dict[str, str]:
     return _load_json_map("elevate_swaps.json")
 
 
-@lru_cache(maxsize=1)
 def load_content_swaps() -> dict[str, str]:
     return _load_json_map("content_swaps.json")
 
 
-@lru_cache(maxsize=1)
 def load_academic_swaps() -> dict[str, str]:
     return _load_json_map("academic_swaps.json")
 
 
-@lru_cache(maxsize=1)
 def load_casual_swaps() -> dict[str, str]:
     return _load_json_map("casual_swaps.json")
 
 
-def _lexicon_for(tone: str, strength: int) -> dict[str, str]:
-    """
-    Merge lexicons by canonical tone (classical fallback only).
-
-    - content_swaps for every tone
-    - Formal: elevate_swaps on top
-    - Academic: elevate + academic_swaps
-    - Casual: preferred + casual downshift (wins over content)
-    - Neutral: preferred wins over content (see preferred_swaps.json); elevate only at Heavy
-    """
-    content = load_content_swaps()
-    tone_l = normalize_tone(tone)
-    if is_academic(tone_l):
-        return {**content, **load_elevate_swaps(), **load_academic_swaps()}
-    if is_elevated(tone_l):  # Formal
-        return {**content, **load_elevate_swaps()}
-    if is_casual(tone_l):
-        return {**content, **load_preferred_swaps(), **load_casual_swaps()}
-    # Neutral: preferred overlays content so plain wording wins
-    merged = {**content, **load_preferred_swaps()}
-    if tone_l == "Neutral" and strength >= 2:
-        for k, v in load_elevate_swaps().items():
-            merged.setdefault(k, v)
-    return merged
-
-
-def _wn_pos(tag: str) -> str | None:
-    if tag.startswith("J"):
-        return "a"
-    if tag.startswith("V"):
-        return "v"
-    if tag.startswith("N"):
-        return "n"
-    if tag.startswith("R"):
-        return "r"
-    return None
-
-
-def _looks_inflected_verb(word: str) -> bool:
-    w = word.lower()
-    return w in _IRREGULAR_PAST or w.endswith(("ing", "ed"))
-
-
-def _is_morph_cousin(original: str, candidate: str) -> bool:
-    """Reject truncated / lookalike forms (climate→clime) that are not real paraphrases."""
-    o, c = original.lower(), candidate.lower()
-    if o.startswith(c) or c.startswith(o):
-        if abs(len(o) - len(c)) <= 3 and min(len(o), len(c)) >= 3:
-            return True
-    # Very high string overlap usually means same stem, not a paraphrase
-    if SequenceMatcher(None, o, c).ratio() >= 0.82:
-        return True
-    return False
-
-
-def _looks_safe_synonym(original: str, candidate: str) -> bool:
-    """Structural synonym safety — works for any word, no domain blocklists.
-
-    Rejects stopwords, lookalike truncations, and shape mismatches.
-    Does NOT require high character similarity (that blocks happy→glad).
-    """
-    o, c = original.lower(), candidate.lower()
-    if c in _STOP_SWAP or c == o:
-        return False
-    if not c.isalpha() or " " in c:
-        return False
-    if len(o) < 4 or len(c) < 4:
-        return False
-    if abs(len(c) - len(o)) > 8:
-        return False
-    if c.endswith(("ish", "ness", "ment")) and not o.endswith(("ish", "ness", "ment")):
-        return False
-    if o in {"first", "second", "third", "fourth", "fifth", "last", "next"}:
-        return False
-    if _is_morph_cousin(o, c):
-        return False
-    return True
-
-
-@lru_cache(maxsize=4096)
-def _wordnet_synonyms(word: str, pos: str | None = None) -> tuple[str, ...]:
-    """WordNet for adjectives/adverbs only — POS-gated, domain-agnostic.
-
-    Nouns/verbs are never queried: WordNet sense inventory is too ambiguous
-    for arbitrary technical/domain text (software→package, etc.).
-    Lexicon JSON covers noun/verb paraphrases deliberately.
-    """
-    if not _ensure_wordnet():
-        return ()
-    if word.lower() in _STOP_SWAP:
-        return ()
-    if _looks_inflected_verb(word):
-        return ()
-    # Nouns & verbs: lexicon only (dynamic for any domain)
-    if pos in {"n", "v"}:
-        return ()
-    from nltk.corpus import wordnet as wn
-
-    # Unknown POS (regex fallback): still adj/adv only — never fall through to nouns
-    order = ["a", "r"]
-    if pos == "r":
-        order = ["r", "a"]
-    elif pos == "a":
-        order = ["a", "r"]
-
-    out: list[str] = []
-    for p in order:
-        synsets = wn.synsets(word, pos=p)
-        if not synsets:
-            continue
-        for lemma in synsets[0].lemmas():
-            name = lemma.name().replace("_", " ").lower()
-            if " " in name:
-                continue
-            if _looks_safe_synonym(word, name) and name not in out:
-                out.append(name)
-        if out:
-            break
-    return tuple(out[:5])
-
-
-@lru_cache(maxsize=4096)
-def _wordnet_synonyms_lemma(lemma: str, pos: str | None = None) -> tuple[str, ...]:
-    """Synonyms for a lemma (used for verbs after spaCy lemmatization)."""
-    if not lemma or len(lemma) < 3:
-        return ()
-    return _wordnet_synonyms(lemma, pos)
-
-
-def _match_case(original: str, replacement: str) -> str:
-    if " " in replacement:
-        parts = replacement.split()
-        if original[:1].isupper():
-            parts[0] = parts[0][:1].upper() + parts[0][1:]
-        return " ".join(parts)
-    if original.isupper():
-        return replacement.upper()
-    if original[:1].isupper():
-        return replacement[:1].upper() + replacement[1:]
-    return replacement
-
-
-def _join_tokens(tokens: list[str]) -> str:
-    text = ""
-    for i, t in enumerate(tokens):
-        if i == 0:
-            text = t
-            continue
-        prev = tokens[i - 1]
-        # Keep contractions/possessives glued: don ' t → don't, cat ' s → cat's
-        if prev.endswith(("'", "’")) or t in {"'", "’"}:
-            text += t
-        elif re.match(r"\w", t):
-            text += " " + t
-        else:
-            text += t
-    return re.sub(r"\s+([,.;:!?])", r"\1", text)
-
-
-def _pick(
-    cands: list[str],
-    tone: str,
-    rng: random.Random,
-    *,
-    original_sentence: str = "",
-    original_word: str = "",
-) -> str:
-    if not cands:
-        return ""
-    tone_l = normalize_tone(tone)
-
-    # MiniLM polish: score full-sentence variants when enabled
-    if ml_polish_enabled() and original_sentence and original_word:
-        variants: list[str] = []
-        low = original_word.lower()
-        for c in cands[:5]:
-            # Replace first case-insensitive whole-word hit
-            pat = re.compile(rf"\b{re.escape(original_word)}\b", re.I)
-            if not pat.search(original_sentence):
-                pat = re.compile(rf"\b{re.escape(low)}\b", re.I)
-            nxt = pat.sub(_match_case(original_word, c), original_sentence, count=1)
-            variants.append(nxt)
-        best_sent = pick_best_candidate(original_sentence, variants, tone=tone_l)
-        if best_sent:
-            for c, variant in zip(cands[:5], variants):
-                if variant == best_sent:
-                    mark_ml_used()
-                    return c
-
-    if is_elevated(tone_l):
-        return max(cands, key=len)
-    if is_casual(tone_l):
-        # Prefer shorter, punchier synonyms
-        return min(cands, key=lambda w: (len(w), w))
-    return rng.choice(cands)
-
-
-def _is_sentence_start(tokens: list[str], index: int) -> bool:
-    if index == 0:
-        return True
-    for j in range(index - 1, -1, -1):
-        t = tokens[j]
-        if t.strip():
-            return t in {".", "!", "?", "…"} or t.endswith((".", "!", "?"))
-    return True
-
-
 def rewrite_sentence_synonyms(
     sentence: str,
-    strength: int,
-    rng: random.Random,
-    *,
     tone: str = "Neutral",
-    force_all_lexicon: bool = False,
+    strength: int = 1,
+    rng=None,
 ) -> str:
-    """
-    Dynamic per-token rewrite for any input:
-    lexicon (content + tone register) then optional WordNet.
-    """
-    lexicon = _lexicon_for(tone, strength)
-    tone_l = normalize_tone(tone)
-    openers = _OPENERS.get(tone_l, {})
-    # Formal/Academic: lexicon-only register. Casual: mostly lexicon (safer downshift).
-    # Neutral: adj/adv WordNet allowed more freely.
-    if is_elevated(tone_l):
-        max_wn = 0
-        wn_rate = 0.0
-    elif is_casual(tone_l):
-        max_wn = {0: 0, 1: 1, 2: 2}.get(strength, 1)
-        wn_rate = {0: 0.0, 1: 0.35, 2: 0.55}.get(strength, 0.35)
-    else:
-        max_wn = {0: 1, 1: 3, 2: 5}.get(strength, 3)
-        wn_rate = {0: 0.4, 1: 0.7, 2: 0.9}.get(strength, 0.7)
-    # Lexicon hits always apply at Normal+; Light usually applies; second pass forces all
-    lex_rate = 1.0 if force_all_lexicon else {0: 0.85, 1: 1.0, 2: 1.0}.get(strength, 1.0)
-
-    def transform(
-        raw: str,
-        pos: str | None,
-        wn_budget: list[int],
-        *,
-        sentence_start: bool,
-        lemma: str | None = None,
-    ) -> str:
-        low = raw.lower()
-        if not raw.isalpha():
-            return raw
-        if sentence_start and low in openers:
-            return openers[low]
-        if low in _STOP_SWAP or len(low) < 4:
-            return raw
-        if low in lexicon and rng.random() < lex_rate:
-            return _match_case(raw, lexicon[low])
-        # WordNet only for adjectives/adverbs (pos already gated inside helper)
-        if pos in {"n", "v"}:
-            return raw
-        if wn_budget[0] <= 0 or rng.random() > wn_rate:
-            return raw
-        if lemma and _looks_inflected_verb(low) and pos == "v":
-            return raw
-        cands = list(_wordnet_synonyms(low, pos))
-        if not cands and lemma and lemma.lower() == low and pos in {None, "a", "r"}:
-            cands = list(_wordnet_synonyms_lemma(lemma, pos))
-        if not cands:
-            return raw
-        wn_budget[0] -= 1
-        return _match_case(
-            raw,
-            _pick(
-                cands,
-                tone,
-                rng,
-                original_sentence=sentence,
-                original_word=raw,
-            ),
-        )
-
-    def _force_min_changes(
-        tokens: list[str],
-        out: list[str],
-        min_changes: int,
-        protected_idx: set[int] | None = None,
-    ) -> list[str]:
-        protected_idx = protected_idx or set()
-        changed = sum(1 for a, b in zip(tokens, out) if a.isalpha() and a != b)
-        if changed >= min_changes:
-            return out
-        for i, tok in enumerate(tokens):
-            if i in protected_idx or not tok.isalpha():
-                continue
-            if out[i] != tok:
-                continue
-            low = tok.lower()
-            if low in _STOP_SWAP or len(low) < 4 or low not in lexicon:
-                continue
-            out[i] = _match_case(tok, lexicon[low])
-            changed += 1
-            if changed >= min_changes:
-                break
-        # Adj/adv WordNet fill only — never nouns/verbs (no domain lists needed)
-        if changed < min_changes and max_wn > 0:
-            for i, tok in enumerate(tokens):
-                if i in protected_idx or not tok.isalpha() or out[i] != tok:
-                    continue
-                low = tok.lower()
-                if low in _STOP_SWAP or len(low) < 4 or _looks_inflected_verb(low):
-                    continue
-                cands = list(_wordnet_synonyms(low, "a"))
-                if not cands:
-                    cands = list(_wordnet_synonyms(low, "r"))
-                if not cands:
-                    continue
-                out[i] = _match_case(
-                    tok,
-                    _pick(
-                        cands,
-                        tone,
-                        rng,
-                        original_sentence=sentence,
-                        original_word=tok,
-                    ),
-                )
-                changed += 1
-                if changed >= min_changes:
-                    break
-        return out
-
-    nlp = get_nlp()
-    spans = _protected_spans(sentence)
-
-    if nlp is None:
-        matches = list(re.finditer(r"\w+|[^\w\s]", sentence, flags=re.UNICODE))
-        tokens = [m.group(0) for m in matches]
-        protected_idx = {
-            i
-            for i, m in enumerate(matches)
-            if _in_protected_span(m.start(), m.end(), spans)
-        }
-        content_n = sum(
-            1
-            for t in tokens
-            if t.isalpha() and t.lower() not in _STOP_SWAP and len(t) >= 4
-        )
-        min_ch = {0: 1, 1: max(2, content_n // 4), 2: max(3, content_n // 3)}.get(
-            strength, 2
-        )
-        if force_all_lexicon:
-            min_ch = max(min_ch, max(2, content_n // 3))
-        budget = [max_wn]
-        out = [
-            tok
-            if i in protected_idx
-            else transform(tok, None, budget, sentence_start=_is_sentence_start(tokens, i))
-            for i, tok in enumerate(tokens)
-        ]
-        out = _force_min_changes(tokens, out, min_ch, protected_idx)
-        return _join_tokens(out)
-
-    doc = nlp(sentence)
-    budget = [max_wn]
-    pieces: list[str] = []
-    toks = list(doc)
-    protected_idx = {
-        i
-        for i, token in enumerate(toks)
-        if _in_protected_span(token.idx, token.idx + len(token.text), spans)
-    }
-    for i, token in enumerate(toks):
-        raw = token.text
-        if i in protected_idx or not token.is_alpha:
-            pieces.append(raw)
-            continue
-        prev_text = toks[i - 1].text if i else ""
-        start = i == 0 or prev_text in {".", "!", "?", "…"}
-        if token.ent_type_ or token.pos_ in {"PROPN", "PRON", "DET", "ADP", "PART"}:
-            low = raw.lower()
-            # Never rewrite closed-class / stop words — even if a lexicon has an entry
-            if low in _STOP_SWAP or token.pos_ in {"PROPN", "PRON", "DET"}:
-                pieces.append(openers[low] if start and low in openers else raw)
-            elif low in lexicon and (force_all_lexicon or rng.random() < lex_rate):
-                pieces.append(_match_case(raw, lexicon[low]))
-            else:
-                pieces.append(openers[low] if start and low in openers else raw)
-            continue
-        if token.pos_ in {"CCONJ", "SCONJ"}:
-            low = raw.lower()
-            if start and low in openers:
-                pieces.append(openers[low])
-            elif low in _STOP_SWAP:
-                pieces.append(raw)
-            elif low in lexicon and low not in {"and", "or", "if"}:
-                pieces.append(_match_case(raw, lexicon[low]))
-            else:
-                pieces.append(raw)
-            continue
-        if token.pos_ in {"VERB", "AUX"}:
-            low = raw.lower()
-            if low in lexicon and (force_all_lexicon or rng.random() < lex_rate):
-                pieces.append(_match_case(raw, lexicon[low]))
-            elif not _looks_inflected_verb(low):
-                pieces.append(
-                    transform(
-                        raw,
-                        "v",
-                        budget,
-                        sentence_start=start,
-                        lemma=token.lemma_,
-                    )
-                )
-            else:
-                pieces.append(raw)
-            continue
-        pieces.append(
-            transform(
-                raw,
-                _wn_pos(token.tag_),
-                budget,
-                sentence_start=start,
-                lemma=token.lemma_,
-            )
-        )
-
-    # Force lexicon fills if barely changed
-    raws = [t.text for t in toks]
-    content_n = sum(
-        1
-        for t in toks
-        if t.is_alpha and t.text.lower() not in _STOP_SWAP and len(t.text) >= 4
-    )
-    min_ch = {0: 1, 1: max(2, content_n // 4), 2: max(3, content_n // 3)}.get(strength, 2)
-    if force_all_lexicon:
-        min_ch = max(min_ch, max(2, content_n // 3))
-    pieces = _force_min_changes(raws, pieces, min_ch, protected_idx)
-    return "".join(p + t.whitespace_ for t, p in zip(doc, pieces)).strip()
+    """No-op unless ALLOW_SYNONYM_REPLACEMENT is explicitly enabled."""
+    _ = (tone, strength, rng)
+    if not ALLOW_SYNONYM_REPLACEMENT:
+        return sentence
+    # Synonym maps removed — structural engine is the primary path.
+    return sentence

requirements.txt

@@ -3,16 +3,19 @@ uvicorn[standard]>=0.30.0
 spacy>=3.7.0,<3.9.0
 # Wheel avoids `python -m spacy download` SSL failures on Spaces / Windows
 https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl
-nltk>=3.9.0
 pydantic>=2.0.0
 python-multipart>=0.0.9
 httpx>=0.27.0
 PyJWT[crypto]>=2.8.0
 python-dotenv>=1.0.0
-# ML polish: MiniLM meaning scoring + generative rewrite (lazy-loaded)
-# Backends: seq2seq (FLAN-T5) or causal chat (SmolLM2) via GENERATIVE_BACKEND
-fastembed>=0.4.2
-torch>=2.2.0
-transformers>=4.40.0
-sentencepiece>=0.2.0
-accelerate>=0.30.0
+pytest>=8.0.0
+
+# Optional: MiniLM safety scoring only (ENGINE_USE_MINILM_SAFETY=true)
+# fastembed>=0.4.2
+# torch>=2.2.0
+# transformers>=4.40.0
+# sentencepiece>=0.2.0
+# accelerate>=0.30.0
+
+# Optional legacy: WordNet synonym path (ALLOW_SYNONYM_REPLACEMENT — unused by structural engine)
+# nltk>=3.9.0

scripts/_probe_reorder_wording.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+import random
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+
+from app.pipeline.orchestrator import _require_visible_rewrite, _try_restructure_unit
+from app.pipeline.restructure import restructure_sentence
+
+samples = [
+    "Ram went to school yesterday happily.",
+    "Nowadays many people are living an unhealthy life because they don't have enough time.",
+    "Online learning makes education more accessible for students in remote areas.",
+    "Sleeping is still an important thing.",
+    "Climate change is causing more extreme weather events around the world.",
+    "Without enough sleep, the brain cannot work properly.",
+]
+
+for s in samples:
+    rr = restructure_sentence(s)
+    print("SRC:", s)
+    print("  rr:", None if not rr else (rr.template_id, rr.text))
+    print("  try:", _try_restructure_unit(s))
+    print("  vis:", _require_visible_rewrite(s, s, "Neutral", random.Random(1)))
+    print()

scripts/test_phase_a_guards.py

@@ -259,6 +259,62 @@ def test_require_visible_keeps_structural_reorder() -> None:
     print("keep structural reorder OK:", out)
 
 
+def test_rejects_degree_adv_junk_reorder() -> None:
+    """Do not ship 'more makes' style degree-adverb reorders."""
+    from app.pipeline.orchestrator import _try_restructure_unit
+    from app.pipeline.restructure import restructure_sentence
+
+    src = "Online learning makes education more accessible for students in remote areas."
+    rr = restructure_sentence(src)
+    if rr is not None:
+        assert "more makes" not in rr.text.lower(), rr.text
+    try_out = _try_restructure_unit(src)
+    if try_out is not None:
+        assert "more makes" not in try_out.lower(), try_out
+    print("degree-adv junk reorder rejected OK:", rr, try_out)
+
+
+def test_because_front_preserves_clauses() -> None:
+    from app.pipeline.restructure import restructure_sentence
+
+    src = (
+        "Nowadays many people are living an unhealthy life "
+        "because they don't have enough time."
+    )
+    rr = restructure_sentence(src)
+    assert rr is not None, "because_front should apply"
+    assert rr.template_id == "because_front"
+    low = rr.text.lower()
+    assert low.startswith("because")
+    assert "unhealthy" in low and "enough time" in low
+    assert "nowadays" in low
+    print("because_front OK:", rr.text)
+
+
+def test_esl_wording_avoids_synonym_thrash() -> None:
+    import random
+
+    from app.pipeline.orchestrator import _require_visible_rewrite
+
+    sleep = "Sleeping is still an important thing."
+    out_s = _require_visible_rewrite(
+        sleep, sleep, "Neutral", random.Random(3), target_sim=0.95
+    )
+    low_s = out_s.lower()
+    assert "endeavor" not in low_s and "nevertheless" not in low_s
+    assert "sleep" in low_s or "sleeping" in low_s
+    print("sleep wording OK:", out_s)
+
+    climate = "Climate change is causing more extreme weather events around the world."
+    out_c = _require_visible_rewrite(
+        climate, climate, "Neutral", random.Random(7), target_sim=0.95
+    )
+    low_c = out_c.lower()
+    assert "uttermost" not in low_c and "upwind" not in low_c
+    assert "climate" in low_c and "weather" in low_c
+    print("climate wording OK:", out_c)
+
+
 def test_rejects_hollow_ons_bills() -> None:
     src = (
         "People should save money for emergencies instead of spending every paycheck."
@@ -574,6 +630,9 @@ if __name__ == "__main__":
     test_complete_wording_rewrite_not_shuffle()
     test_restructure_time_only_and_manner()
     test_require_visible_keeps_structural_reorder()
+    test_rejects_degree_adv_junk_reorder()
+    test_because_front_preserves_clauses()
+    test_esl_wording_avoids_synonym_thrash()
     test_rejects_hollow_ons_bills()
     test_assembly_preserves_unit_count()
     test_live_bug_patterns_rejected()

tests/conftest.py

@@ -0,0 +1,10 @@
+"""Pytest configuration."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))

tests/test_engine_long.py

@@ -0,0 +1,88 @@
+"""Long-document batching and safety tests."""
+
+from __future__ import annotations
+
+import os
+
+os.environ.setdefault("LANGUAGE_TOOL_ENABLED", "false")
+os.environ.setdefault("LANGUAGE_TOOL_URL", "")
+os.environ.setdefault("GRAMMAR_FIX_INPUT", "false")
+os.environ.setdefault("GRAMMAR_FIX_OUTPUT", "false")
+os.environ.setdefault("ENGINE_USE_MINILM_SAFETY", "false")
+
+from app.engine.normalize import detect_blocks, normalize_text
+from app.engine.orchestrator import rewrite_document
+from app.engine.safety import check_safety
+from app.engine.segment import iter_paragraph_batches, word_count
+
+
+def _make_long_doc(target_words: int = 10500) -> str:
+    templates = [
+        "Ram went to school yesterday happily.",
+        "The committee approved the budget today carefully.",
+        "Workers cleaned the factory floor this morning quickly.",
+        "Students completed the assignment last night diligently.",
+        "Although rain fell heavily, traffic moved slowly through town.",
+        "# Section Heading\n\n",
+        "- Keep this list item unchanged forever.\n",
+        "She sent the report to https://files.example.org/a.pdf yesterday.\n\n",
+    ]
+    parts: list[str] = []
+    while word_count("\n\n".join(parts)) < target_words:
+        for t in templates:
+            parts.append(t.strip())
+            if word_count("\n\n".join(parts)) >= target_words:
+                break
+    return "\n\n".join(parts)
+
+
+def test_long_document_batching():
+    doc = _make_long_doc(10500)
+    assert word_count(doc) >= 10000
+    result = rewrite_document(doc, batch_paras=15)
+    assert result.stats.batches >= 2
+    assert result.input_words >= 10000
+    assert result.stats.sentences > 50
+    # Paragraph-ish structure preserved (blank-line separated blocks)
+    assert result.text.count("\n\n") >= 10
+    assert result.stats.seconds < 180  # CPU smoke budget
+
+
+def test_bibliography_passthrough():
+    src = (
+        "Ram went to school yesterday happily.\n\n"
+        "References\n\n"
+        "Smith, J. (2020). A paper about schools. Journal of Education.\n"
+        "Doe, A. (2019). Another citation here with many words about learning."
+    )
+    result = rewrite_document(src)
+    assert "Smith, J." in result.text
+    assert "References" in result.text
+
+
+def test_safety_rejects_entity_loss():
+    original = "Alice met Bob yesterday at the park."
+    bad = "Someone met a friend yesterday at the park."
+    safety = check_safety(original, bad, use_minilm=False)
+    assert not safety.ok
+    assert any(r.startswith("entity:") or r == "negation" for r in safety.reasons) or (
+        "entity" in " ".join(safety.reasons)
+    )
+
+
+def test_safety_rejects_negation_flip():
+    original = "Ram did not go to school yesterday."
+    bad = "Ram did go to school yesterday."
+    safety = check_safety(original, bad, use_minilm=False)
+    assert not safety.ok
+    assert "negation" in safety.reasons or "polarity" in safety.reasons
+
+
+def test_normalize_and_blocks():
+    raw = "Hello   world.\n\n# Title\n\n- item one\n\n```\ncode\n```"
+    norm = normalize_text(raw)
+    blocks = detect_blocks(norm)
+    kinds = {b.kind for b in blocks}
+    assert "heading" in kinds or "paragraph" in kinds
+    batches = list(iter_paragraph_batches(blocks, batch_paras=2))
+    assert batches

tests/test_engine_short.py

@@ -0,0 +1,120 @@
+"""Short-document structural rewrite tests."""
+
+from __future__ import annotations
+
+import os
+
+# Keep tests offline and fast — no LanguageTool / MiniLM
+os.environ.setdefault("LANGUAGE_TOOL_ENABLED", "false")
+os.environ.setdefault("LANGUAGE_TOOL_URL", "")
+os.environ.setdefault("GRAMMAR_FIX_INPUT", "false")
+os.environ.setdefault("GRAMMAR_FIX_OUTPUT", "false")
+os.environ.setdefault("ENGINE_USE_MINILM_SAFETY", "false")
+os.environ.setdefault("ALLOW_SYNONYM_REPLACEMENT", "false")
+
+from app.engine.classify import classify_sentence
+from app.engine.orchestrator import rewrite_document
+from app.engine.plan import build_plan
+from app.engine.rewrite import generate_from_plan
+from app.pipeline.orchestrator import rewrite_text
+
+
+def test_ram_school_reorder():
+    src = "Ram went to school yesterday happily."
+    result = rewrite_document(src)
+    out = result.text.strip().rstrip(".")
+    # Accept either full template or time-front
+    assert "Yesterday" in result.text
+    assert "Ram" in result.text
+    assert "school" in result.text.lower()
+    assert "happily" in result.text.lower()
+    assert "went" in result.text.lower()
+    # Should not be identical
+    assert result.text.strip().lower() != src.strip().lower()
+    rewritten = [s for s in result.sentences if s.status == "rewritten"]
+    assert rewritten, f"expected rewrite, got {result.sentences}"
+
+
+def test_negation_preserved():
+    src = "Ram did not go to school yesterday."
+    result = rewrite_document(src)
+    low = result.text.lower()
+    assert "not" in low or "n't" in low
+    assert "ram" in low
+
+
+def test_entity_preserved():
+    src = "Alice visited Paris yesterday happily."
+    result = rewrite_document(src)
+    assert "Alice" in result.text
+    assert "Paris" in result.text
+
+
+def test_url_and_email_passthrough():
+    src = (
+        "Please read https://example.com/docs carefully today.\n\n"
+        "Contact support at help@example.com for details."
+    )
+    result = rewrite_document(src)
+    assert "https://example.com/docs" in result.text
+    assert "help@example.com" in result.text
+
+
+def test_heading_and_list_untouched():
+    src = (
+        "# Introduction\n\n"
+        "Ram went to school yesterday happily.\n\n"
+        "- First bullet item here\n"
+        "- Second bullet item here"
+    )
+    result = rewrite_document(src)
+    assert "# Introduction" in result.text or "Introduction" in result.text
+    assert "- First bullet item here" in result.text
+    assert "- Second bullet item here" in result.text
+
+
+def test_code_block_untouched():
+    src = (
+        "Ram went to school yesterday happily.\n\n"
+        "```python\nprint('hello')\n```\n\n"
+        "Bob walked home today slowly."
+    )
+    result = rewrite_document(src)
+    assert "```python" in result.text
+    assert "print('hello')" in result.text
+
+
+def test_complex_sentence_skipped():
+    src = "Although the weather was bad, Ram went to school yesterday."
+    result = rewrite_document(src)
+    assert classify_sentence(src) == "complex"
+    # Should remain unchanged (skipped)
+    assert any(s.status == "skipped" for s in result.sentences)
+    assert "Although" in result.text
+
+
+def test_structured_output_fields():
+    src = "Ram went to school yesterday happily."
+    result = rewrite_document(src)
+    assert result.sentences
+    assert result.mapping
+    assert result.stats.sentences >= 1
+    assert result.engine == "structural-reorder"
+
+
+def test_rewrite_text_shim():
+    src = "Ram went to school yesterday happily."
+    result = rewrite_text(src, tone="Neutral", strength=1)
+    assert result.pipeline_mode == "structural"
+    assert result.engine == "structural-reorder"
+    assert result.sentences is not None
+    assert result.stats is not None
+
+
+def test_plan_then_generate():
+    src = "Ram went to school yesterday happily."
+    plan = build_plan(src)
+    assert plan.safe
+    text = generate_from_plan(plan)
+    assert text
+    assert "Yesterday" in text or "happily" in text.lower()