Upload 105 files

app/engine/normalize/__init__.py

@@ -295,7 +295,9 @@ def mask_protected_spans(text: str) -> tuple[str, dict[str, str]]:
 
     def _sub(pattern: re.Pattern[str], label: str, s: str) -> str:
         def repl(m: re.Match[str]) -> str:
-            key = f"__PROT_{label}_{counter['n']}__"
+            # Alphanumeric placeholders stay as one token in spaCy. Underscore
+            # placeholders can be split and lose their trailing delimiter.
+            key = f"ZZPROTECTED{label}{counter['n']}ZZ"
             mapping[key] = m.group(0)
             counter["n"] += 1
             return key
@@ -307,7 +309,7 @@ def mask_protected_spans(text: str) -> tuple[str, dict[str, str]]:
     out = _sub(_EMAIL, "EMAIL", out)
 
     def path_repl(m: re.Match[str]) -> str:
-        key = f"__PROT_PATH_{counter['n']}__"
+        key = f"ZZPROTECTEDPATH{counter['n']}ZZ"
         mapping[key] = m.group(0)
         counter["n"] += 1
         return key

app/engine/orchestrator.py

@@ -108,10 +108,9 @@ def _process_sentence(
         generated = generate_from_plan(plan, template_id=cand.template_id)
         if not generated:
             continue
-        repaired = repair_sentence(generated) if GRAMMAR_FIX_OUTPUT else generated
         safety = check_safety(
             original,
-            repaired,
+            generated,
             min_meaning=safety_min,
             min_confidence=min_confidence,
             use_minilm=use_minilm,
@@ -121,6 +120,26 @@ def _process_sentence(
         if not safety.ok:
             stats.bump(safety.reasons[0] if safety.reasons else "safety")
             continue
+        repaired = repair_sentence(generated) if GRAMMAR_FIX_OUTPUT else generated
+        if repaired != generated:
+            repaired_safety = check_safety(
+                original,
+                repaired,
+                min_meaning=safety_min,
+                min_confidence=min_confidence,
+                use_minilm=use_minilm,
+                protected_entities=plan.slots.entities if plan.slots else (),
+                protected_auxiliaries=plan.slots.auxiliaries if plan.slots else (),
+                structural_validation=False,
+            )
+            if not repaired_safety.ok:
+                stats.bump(
+                    repaired_safety.reasons[0]
+                    if repaired_safety.reasons
+                    else "grammar_safety"
+                )
+                continue
+            safety = repaired_safety
         conf = min(cand.confidence, safety.confidence + 0.2)
         best = SentenceRecord(
             index=index,

app/engine/parse/__init__.py

@@ -120,9 +120,6 @@ def extract_slots(text: str) -> SentenceSlots:
     """Parse subject/verb/object/time/place/manner/negation/entities via spaCy."""
     raw = (text or "").strip()
     slots = SentenceSlots(text=raw, sentence_type=classify_sentence(raw))
-    if slots.sentence_type not in {"simple_declarative", "compound"}:
-        slots.reasons.append(f"skip:{slots.sentence_type}")
-        return slots
 
     nlp = get_nlp()
     if nlp is None:
@@ -136,6 +133,10 @@ def extract_slots(text: str) -> SentenceSlots:
             "PERSON", "ORG", "GPE", "LOC", "DATE", "TIME", "MONEY", "PERCENT", "CARDINAL",
         }
     ]
+    if slots.sentence_type not in {"simple_declarative", "compound"}:
+        slots.auxiliaries = [token.text for token in doc if token.pos_ == "AUX"]
+        slots.reasons.append(f"skip:{slots.sentence_type}")
+        return slots
 
     root = next((t for t in doc if t.dep_ == "ROOT" and t.pos_ in {"VERB", "AUX"}), None)
     if root is None:
@@ -243,7 +244,11 @@ def extract_slots(text: str) -> SentenceSlots:
         # Duration under for — skip; bare adjunct DATE/TIME — keep
         if under and ent.label_ == "TIME":
             continue
-        time_parts.append(ent.text)
+        governing_prep = ent.root.head
+        if governing_prep.dep_ == "prep" and governing_prep.pos_ == "ADP":
+            time_parts.append(_span_text(_subtree_tokens(governing_prep)))
+        else:
+            time_parts.append(ent.text)
 
     for t in doc:
         if _is_frontable_adv(t, root):
@@ -289,6 +294,18 @@ def extract_slots(text: str) -> SentenceSlots:
         slots.manner = ""
     if slots.manner and slots.time and slots.manner.lower() in slots.time.lower():
         slots.manner = ""
+    residual_adverbs = [
+        token
+        for token in early_manner
+        if token.text.lower() != slots.manner.lower()
+        and not (_is_temporal_tok(token) or token.ent_type_ in {"DATE", "TIME"})
+    ]
+    if residual_adverbs:
+        phrase_tokens = sorted(
+            {token.i: token for token in [*verb_toks, *residual_adverbs]}.values(),
+            key=lambda token: token.i,
+        )
+        slots.verb_phrase = _span_text(phrase_tokens)
 
     # If place was absorbed into object, keep object but allow place move
     if slots.place and slots.object and slots.place in slots.object:

app/engine/plan/__init__.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 from app.engine.classify import classify_sentence, is_rewriteable_type
 from app.engine.models import RewritePlan, SentenceSlots, TemplateCandidate
 from app.engine.parse import extract_slots
-from app.engine.templates import rank_templates
+from app.engine.templates import can_swap_initial_subordinate, rank_templates
 
 
 def build_plan(text: str, *, min_confidence: float = 0.55) -> RewritePlan:
@@ -26,6 +26,19 @@ def build_plan(text: str, *, min_confidence: float = 0.55) -> RewritePlan:
             fixed_spans=[],
         )
 
+    if kind == "complex" and can_swap_initial_subordinate(raw):
+        slots = extract_slots(raw)
+        slots.confidence = 0.76
+        return RewritePlan(
+            safe=True,
+            template_id="complex_clause_swap",
+            candidates=[TemplateCandidate("complex_clause_swap", 0.76)],
+            confidence=0.76,
+            slots=slots,
+            movable=["subordinate_clause"],
+            fixed_spans=list(slots.entities),
+        )
+
     if not is_rewriteable_type(kind):
         return RewritePlan(
             safe=False,

app/engine/rewrite/__init__.py

@@ -5,7 +5,11 @@ from __future__ import annotations
 import re
 
 from app.engine.models import RewritePlan
-from app.engine.templates import fill_template, try_because_front
+from app.engine.templates import (
+    fill_template,
+    try_because_front,
+    try_complex_clause_swap,
+)
 
 
 def _content_tokens(text: str) -> list[str]:
@@ -72,6 +76,8 @@ def generate_from_plan(
 
     if tid == "because_front":
         return try_because_front(raw)
+    if tid == "complex_clause_swap":
+        return try_complex_clause_swap(raw)
 
     if plan.slots is None:
         return None

app/engine/safety/__init__.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import re
 from collections.abc import Iterable
 from dataclasses import dataclass
+from difflib import SequenceMatcher
 
 from app.pipeline.candidate_validator import validate_candidate
 from app.pipeline.meaning_safety import polarity_safe
@@ -83,6 +84,7 @@ def check_safety(
     use_minilm: bool = False,
     protected_entities: Iterable[str] | None = None,
     protected_auxiliaries: Iterable[str] | None = None,
+    structural_validation: bool = True,
 ) -> SafetyResult:
     """Lightweight similarity/safety gate between original and rewrite."""
     reasons: list[str] = []
@@ -119,26 +121,33 @@ def check_safety(
     if not _tense_aux_ok(o, c, protected_auxiliaries):
         reasons.append("tense")
 
-    # Structural reorder may be near-copy in surface ratio; relax max_surface
-    vr = validate_candidate(
-        o,
-        c,
-        min_meaning=min_meaning if use_minilm else 0.0,
-        max_surface=0.995,
-        min_surface=0.20,
-    )
-    # Filter validator reasons that fight structural reorder
-    ignore = {"too_similar", "identical"}
-    for r in vr.reasons:
-        if r in ignore:
-            continue
-        if r.startswith("meaning:") and not use_minilm:
-            continue
-        if r not in reasons:
-            reasons.append(r)
+    if structural_validation:
+        # Structural reorder may be near-copy in surface ratio; relax max_surface
+        vr = validate_candidate(
+            o,
+            c,
+            min_meaning=min_meaning if use_minilm else 0.0,
+            max_surface=0.995,
+            min_surface=0.20,
+        )
+        # Filter validator reasons that fight structural reorder
+        ignore = {"too_similar", "identical"}
+        for r in vr.reasons:
+            if r in ignore:
+                continue
+            if r.startswith("meaning:") and not use_minilm:
+                continue
+            if r not in reasons:
+                reasons.append(r)
+        surface_sim = vr.surface_sim
+        meaning = vr.meaning
+    else:
+        # Grammar repair may legitimately change inflection or spelling. Recheck
+        # only hard invariants while still reporting a lightweight similarity.
+        surface_sim = SequenceMatcher(None, o.lower(), c.lower()).ratio()
+        meaning = surface_sim
 
     # Optional MiniLM meaning score when enabled
-    meaning = vr.meaning
     if use_minilm:
         try:
             from app.pipeline.minilm import score_candidate
@@ -151,7 +160,10 @@ def check_safety(
         except Exception:
             pass
 
-    confidence = max(0.0, min(1.0, (meaning + (1.0 - abs(vr.surface_sim - 0.7))) / 2))
+    confidence = max(
+        0.0,
+        min(1.0, (meaning + (1.0 - abs(surface_sim - 0.7))) / 2),
+    )
     if reasons:
         confidence = min(confidence, 0.4)
 
@@ -177,7 +189,7 @@ def check_safety(
     }
     if hard:
         ok = False
-    elif reasons and vr.surface_sim >= 0.35:
+    elif reasons and surface_sim >= 0.35:
         # Soft validator noise on reorders — accept if content preserved
         ok = True
         confidence = max(confidence, 0.6)
@@ -186,6 +198,6 @@ def check_safety(
         ok=ok,
         confidence=confidence,
         reasons=reasons,
-        surface_sim=vr.surface_sim,
+        surface_sim=surface_sim,
         meaning=float(meaning or 0.0),
     )

app/engine/templates/__init__.py

@@ -3,8 +3,10 @@
 from __future__ import annotations
 
 import re
+from functools import lru_cache
 
 from app.engine.models import SentenceSlots, TemplateCandidate
+from app.pipeline.nlp import get_nlp
 
 
 def rank_templates(slots: SentenceSlots) -> list[TemplateCandidate]:
@@ -86,6 +88,63 @@ def _terminal(text: str) -> str:
     return "."
 
 
+@lru_cache(maxsize=2048)
+def _initial_subordinate_parts(text: str) -> tuple[str, str, str, bool] | None:
+    """Return an initial parsed adverbial clause and its main clause."""
+    raw = (text or "").strip()
+    if "," not in raw:
+        return None
+    nlp = get_nlp()
+    if nlp is None:
+        return None
+    try:
+        doc = nlp(raw)
+    except Exception:
+        return None
+    comma = next((token for token in doc if token.text == ","), None)
+    root = next((token for token in doc if token.dep_ == "ROOT"), None)
+    if comma is None or root is None or root.i <= comma.i:
+        return None
+    clause = next(
+        (
+            token
+            for token in doc
+            if token.dep_ == "advcl"
+            and min(part.i for part in token.subtree) == 0
+            and max(part.i for part in token.subtree) < comma.i
+            and any(
+                part.dep_ == "mark" or part.pos_ == "SCONJ"
+                for part in token.subtree
+            )
+        ),
+        None,
+    )
+    if clause is None:
+        return None
+
+    end = _terminal(raw)
+    core = raw[:-1].rstrip() if raw.endswith((".", "!", "?")) else raw
+    subordinate, main = (part.strip() for part in core.split(",", 1))
+    if len(subordinate.split()) < 3 or len(main.split()) < 3:
+        return None
+    return subordinate, main, end, doc[0].dep_ == "mark"
+
+
+def can_swap_initial_subordinate(text: str) -> bool:
+    return _initial_subordinate_parts(text) is not None
+
+
+def try_complex_clause_swap(text: str) -> str | None:
+    """Move a parsed initial adverbial clause behind the main clause."""
+    parts = _initial_subordinate_parts(text)
+    if parts is None:
+        return None
+    subordinate, main, end, needs_comma = parts
+    continuation = subordinate[0].lower() + subordinate[1:]
+    separator = ", " if needs_comma else " "
+    return f"{_cap(main)}{separator}{continuation}{end}"
+
+
 def _manner_before_verb(slots: SentenceSlots) -> bool:
     return not slots.verb_starts_with_aux