Upload 146 files

app/engine/classify/__init__.py

@@ -1,15 +1,10 @@
-"""Sentence type classification — only safe types enter the rewrite path."""
+"""Sentence type classification — spaCy-driven when available."""
 
 from __future__ import annotations
 
 import re
 
-_SUBORDINATORS = frozenset(
-    """
-    although though while whilst whereas unless until since
-    if when whenever wherever whether before after
-    """.split()
-)
+from app.pipeline.nlp import get_nlp
 
 _REWRITEABLE = frozenset({"simple_declarative", "compound", "because_clause"})
 
@@ -34,27 +29,58 @@ def classify_sentence(text: str) -> str:
     if len(t.split()) > 45:
         return "too_long"
 
-    low = t.lower()
-    if re.search(r"\bbecause\b", low):
-        return "because_clause"
+    nlp = get_nlp()
+    if nlp is not None:
+        return _classify_spacy(t, nlp)
+
+    return _classify_regex(t)
+
 
-    # Coordinating compound (safe if short enough)
-    if re.search(r"\b(and|but|or|so|yet)\b", low) and "," in t:
-        # Still complex if subordinators present
-        for sub in _SUBORDINATORS:
-            if re.search(rf"\b{re.escape(sub)}\b", low):
+def _classify_spacy(text: str, nlp) -> str:
+    doc = nlp(text)
+    # because as mark / SCONJ
+    for t in doc:
+        if t.lemma_.lower() == "because" and t.pos_ in {"SCONJ", "ADP"}:
+            return "because_clause"
+    # Subordinate clauses via mark / advcl / SCONJ
+    for t in doc:
+        if t.dep_ == "mark" and t.head.dep_ in {"advcl", "acl"}:
+            if t.lemma_.lower() != "because":
                 return "complex"
-        if len(t.split()) <= 28:
+        if t.pos_ == "SCONJ" and t.lemma_.lower() != "because":
+            if t.head.dep_ in {"advcl", "acl", "ROOT"} or t.dep_ == "mark":
+                return "complex"
+    # Relative clauses on longer sentences are unsafe to slot-rebuild
+    if any(t.dep_ == "relcl" for t in doc) and len(text.split()) >= 10:
+        return "complex"
+    # Coordinating compound with comma
+    if "," in text and any(t.dep_ == "cc" and t.head.dep_ in {"conj", "ROOT"} for t in doc):
+        if len(text.split()) <= 28:
             return "compound"
         return "complex"
+    return "simple_declarative"
 
-    for sub in _SUBORDINATORS:
-        if re.search(rf"\b{re.escape(sub)}\b", low):
-            return "complex"
 
-    if re.search(r"\b(who|whom|whose|which|that)\b", low) and len(t.split()) > 12:
+def _classify_regex(text: str) -> str:
+    low = text.lower()
+    if re.search(r"\bbecause\b", low):
+        return "because_clause"
+    if re.search(r"\b(and|but|or|so|yet)\b", low) and "," in text:
+        if re.search(
+            r"\b(although|though|while|whilst|whereas|unless|until|since|if|when|"
+            r"whenever|wherever|whether|before|after)\b",
+            low,
+        ):
+            return "complex"
+        return "compound" if len(text.split()) <= 28 else "complex"
+    if re.search(
+        r"\b(although|though|while|whilst|whereas|unless|until|since|if|when|"
+        r"whenever|wherever|whether|before|after)\b",
+        low,
+    ):
+        return "complex"
+    if re.search(r"\b(who|whom|whose|which|that)\b", low) and len(text.split()) > 12:
         return "complex"
-
     return "simple_declarative"
 
 

app/engine/consistency/__init__.py

@@ -1,4 +1,4 @@
-"""Document-level consistency pass after sentence rewrite."""
+"""Document-level consistency — spaCy PROPN only, no word lists."""
 
 from __future__ import annotations
 
@@ -6,45 +6,28 @@ import re
 from collections import defaultdict
 
 from app.engine.models import SentenceRecord
+from app.pipeline.nlp import get_nlp
 
-# Words that are often sentence-initial — never treat as entity canon targets
-_COMMON_STARTS = frozenset(
-    """
-    the a an and or but if when as at by for in on to of with from
-    this that these those it its they them he she we you i my our your
-    many most some any all each every other such own same too not
-    reading writing sleeping walking running making taking getting
-    students experts people workers researchers managers teachers
-    yesterday today tomorrow earlier later recently now
-    because although however therefore moreover furthermore
-    """.split()
-)
 
-
-def _mid_sentence_propers(text: str) -> list[str]:
-    """Capitalized tokens that appear mid-sentence (likely real names)."""
-    raw = text or ""
-    out: list[str] = []
-    parts = re.split(r"(?<=[.!?])\s+", raw)
-    for part in parts:
-        tokens = re.findall(r"\b[A-Za-z][A-Za-z0-9'-]*\b", part)
-        for tok in tokens[1:]:
-            if tok[0].isupper() and tok.lower() not in _COMMON_STARTS:
-                out.append(tok)
-    return out
+def _proper_from_text(text: str) -> list[str]:
+    nlp = get_nlp()
+    if nlp is None:
+        return []
+    try:
+        doc = nlp(text or "")
+        return [t.text for t in doc if t.pos_ == "PROPN"]
+    except Exception:
+        return []
 
 
 def canonicalize_entities(text: str, records: list[SentenceRecord]) -> str:
-    """Normalize true proper-noun spelling; ignore sentence-initial capitals."""
+    """Normalize PROPN spelling to first-seen form."""
     canon: dict[str, str] = {}
     for rec in records:
-        for name in _mid_sentence_propers(rec.original):
+        for name in _proper_from_text(rec.original):
             key = name.lower()
-            if key in _COMMON_STARTS:
-                continue
             if key not in canon:
                 canon[key] = name
-
     if not canon:
         return text
 
@@ -59,13 +42,14 @@ def canonicalize_entities(text: str, records: list[SentenceRecord]) -> str:
 
 
 def suppress_template_streaks(records: list[SentenceRecord]) -> list[SentenceRecord]:
-    """If two consecutive rewrites used the same time-front template, revert the second."""
     time_fronts = {
         "time_subj_manner_verb_place",
         "time_subj_verb_place",
         "time_subj_verb_object",
         "time_subj_verb_object_place",
         "time_front",
+        "place_front",
+        "adv_subj_verb_object",
     }
     out: list[SentenceRecord] = []
     prev_tid = ""
@@ -75,18 +59,19 @@ def suppress_template_streaks(records: list[SentenceRecord]) -> list[SentenceRec
             and rec.template_id in time_fronts
             and prev_tid == rec.template_id
         ):
-            reverted = SentenceRecord(
-                index=rec.index,
-                original=rec.original,
-                rewritten=rec.original,
-                confidence=rec.confidence,
-                status="reverted",
-                template_id="",
-                sentence_type=rec.sentence_type,
-                reasons=rec.reasons + ["template_streak"],
-                block_index=rec.block_index,
+            out.append(
+                SentenceRecord(
+                    index=rec.index,
+                    original=rec.original,
+                    rewritten=rec.original,
+                    confidence=rec.confidence,
+                    status="reverted",
+                    template_id="",
+                    sentence_type=rec.sentence_type,
+                    reasons=rec.reasons + ["template_streak"],
+                    block_index=rec.block_index,
+                )
             )
-            out.append(reverted)
             prev_tid = ""
             continue
         out.append(rec)
@@ -98,15 +83,13 @@ def apply_consistency(
     text: str,
     records: list[SentenceRecord],
 ) -> tuple[str, list[SentenceRecord]]:
-    """Run light document-level consistency fixes."""
     adjusted = suppress_template_streaks(records)
-    fixed_text = canonicalize_entities(text, adjusted)
-    return fixed_text, adjusted
+    return canonicalize_entities(text, adjusted), adjusted
 
 
 def entity_frequency(records: list[SentenceRecord]) -> dict[str, int]:
     freq: dict[str, int] = defaultdict(int)
     for rec in records:
-        for name in _mid_sentence_propers(rec.original):
+        for name in _proper_from_text(rec.original):
             freq[name] += 1
     return dict(freq)

app/engine/grammar/__init__.py

@@ -5,26 +5,33 @@ from __future__ import annotations
 import re
 
 from app.pipeline.grammar_fix import correct_text
-
-
-_COMMON_AFTER_COMMA = frozenset(
-    """
-    the a an and or but if when as at by for in on to of with from
-    this that these those it its they them he she we you i my our your
-    many most some any all each every other such reading writing sleeping
-    students experts people workers researchers managers teachers
-    she he they we it
-    """.split()
-)
+from app.pipeline.nlp import get_nlp
 
 
 def _fix_intro_comma_caps(text: str) -> str:
-    """Lowercase common words right after an introductory comma; keep names."""
+    """Lowercase non-PROPN tokens right after an introductory comma (spaCy POS)."""
+    nlp = get_nlp()
 
     def repl(m: re.Match[str]) -> str:
         word = m.group(1)
-        if word.lower() in _COMMON_AFTER_COMMA:
-            return ", " + word[0].lower() + word[1:]
+        if nlp is not None:
+            try:
+                tok = nlp(word)[0]
+                if tok.pos_ == "PROPN":
+                    return m.group(0)
+                if tok.pos_ in {
+                    "DET",
+                    "PRON",
+                    "ADP",
+                    "SCONJ",
+                    "CCONJ",
+                    "ADV",
+                    "NOUN",
+                    "ADJ",
+                }:
+                    return ", " + word[0].lower() + word[1:]
+            except Exception:
+                pass
         return m.group(0)
 
     return re.sub(r",\s+([A-Z][a-zA-Z']*)", repl, text)

app/engine/parse/__init__.py

@@ -1,4 +1,4 @@
-"""Dependency parsing — spaCy-first slot extraction (no synonym tables)."""
+"""Dependency parsing — spaCy-driven slots (no synonym or domain word lists)."""
 
 from __future__ import annotations
 
@@ -8,30 +8,6 @@ from app.engine.classify import classify_sentence
 from app.engine.models import SentenceSlots
 from app.pipeline.nlp import get_nlp
 
-# Closed-class helpers only (spaCy NER/deps preferred for open-class time/place)
-_TIME_WORDS = frozenset(
-    """
-    yesterday today tomorrow earlier later recently now tonight
-    monday tuesday wednesday thursday friday saturday sunday
-    morning afternoon evening night
-    """.split()
-)
-_FREQ_TIME = frozenset(
-    """
-    everyday daily nightly weekly monthly yearly annually
-    """.split()
-)
-_DISCOURSE_TIME = frozenset({"nowadays"})
-_DEGREE_ADVS = frozenset(
-    """
-    more most less least much many very really quite just also still even
-    only rather pretty fairly so too enough almost nearly already ever never
-    not n't however nevertheless furthermore moreover
-    """.split()
-)
-_PLACE_PREPS = frozenset("to at in on into onto from toward towards".split())
-_DURATION_PREPS = frozenset({"for", "over", "within", "during"})
-
 
 def _span_text(tokens: list) -> str:
     if not tokens:
@@ -43,36 +19,71 @@ def _subtree_tokens(token) -> list:
     return sorted(token.subtree, key=lambda t: t.i)
 
 
-def _is_manner_adv(word: str) -> bool:
-    low = (word or "").lower().strip()
-    if not low or low in _DEGREE_ADVS or low in _DISCOURSE_TIME:
-        return False
-    return low.endswith("ly") and len(low) >= 4
+def _under_duration_prep(token) -> bool:
+    """True if token sits under a duration-like preposition (for/over/within/during)."""
+    cur = token
+    while cur is not None and cur.head != cur:
+        if cur.dep_ == "prep" and cur.lemma_.lower() in {"for", "over", "within", "during"}:
+            # Duration if pobj/subtree looks numeric or has TIME unit via ent/morph
+            return True
+        cur = cur.head
+    return False
 
 
-def _is_duration_span(span: str, full_text: str) -> bool:
-    """Durations inside for/over/in are not frontable sentence time."""
-    s = (span or "").strip()
-    if not s:
-        return False
-    low = s.lower()
-    if re.search(r"\b(minute|minutes|hour|hours|second|seconds)\b", low):
+def _is_duration_ent(ent, doc_text: str) -> bool:
+    """DATE/TIME under for/over, or minute/hour-style spans — not sentence adjuncts."""
+    text = ent.text.strip()
+    if re.search(
+        rf"\b(for|over|within|during)\s+{re.escape(text)}\b",
+        doc_text,
+        flags=re.I,
+    ):
         return True
-    if re.match(r"at\s+least\b", low):
+    for t in ent:
+        if _under_duration_prep(t):
+            # Allow calendar adjuncts like "for Monday" rarely; block if CARDINAL/TIME unitish
+            if ent.label_ in {"TIME", "DATE"} and re.search(
+                r"\b(minute|minutes|hour|hours|second|seconds)\b",
+                text,
+                flags=re.I,
+            ):
+                return True
+            if re.match(r"at\s+least\b", text, flags=re.I):
+                return True
+            if ent.label_ == "TIME" and _under_duration_prep(t):
+                return True
+    return False
+
+
+def _is_frontable_adv(tok, root) -> bool:
+    """ADV modifiers that can move — derived from POS/DEP, not word lists."""
+    if tok.pos_ != "ADV":
+        return False
+    if tok.dep_ not in {"advmod", "npadvmod"}:
+        return False
+    if tok.lemma_.lower() in {"not", "n't"} or tok.dep_ == "neg":
+        return False
+    # Degree on adjectives/adverbs: "very important", "really useful"
+    if tok.head.pos_ in {"ADJ", "ADV"} and tok.head.i != root.i:
+        return False
+    return True
+
+
+def _is_temporal_tok(tok) -> bool:
+    if tok.ent_type_ in {"DATE", "TIME"}:
         return True
-    if re.search(rf"\b(for|over|within|during)\s+{re.escape(s)}\b", full_text, flags=re.I):
+    # spaCy morph sometimes marks Tense; prefer ent + npadvmod of DATE-like
+    if tok.dep_ in {"npadvmod", "advmod"} and tok.ent_type_ in {"DATE", "TIME"}:
         return True
+    if tok.dep_ == "npadvmod" and tok.pos_ in {"NOUN", "PROPN"}:
+        # "yesterday"/"today" often NOUN+npadvmod without ent in sm model
+        if tok.head.pos_ in {"VERB", "AUX"}:
+            return True
     return False
 
 
 def _dedupe_spans(spans: list[str]) -> list[str]:
-    """Prefer longer spans; drop ones contained in another."""
-    cleaned = []
-    for s in spans:
-        s = (s or "").strip()
-        if not s:
-            continue
-        cleaned.append(s)
+    cleaned = [s.strip() for s in spans if s and s.strip()]
     cleaned.sort(key=len, reverse=True)
     kept: list[str] = []
     for s in cleaned:
@@ -85,24 +96,28 @@ def _dedupe_spans(spans: list[str]) -> list[str]:
     return kept
 
 
-def _extract_freq_time(text: str) -> str:
-    """Frontable frequency adverbials (every day / everyday / each morning)."""
-    raw = text or ""
-    m = re.search(
-        r"\b(every|each)\s+(day|night|morning|evening|week|month|year)\b",
-        raw,
-        flags=re.I,
-    )
-    if m:
-        return m.group(0)
-    m2 = re.search(r"\b(everyday|daily|nightly|weekly|monthly|annually)\b", raw, flags=re.I)
-    if m2:
-        return m2.group(1)
+def _freq_from_doc(doc) -> str:
+    """every/each + noun via dependency structure (not a word list of weekdays)."""
+    for tok in doc:
+        if tok.pos_ == "DET" and tok.lemma_.lower() in {"every", "each"}:
+            head = tok.head
+            if head.pos_ in {"NOUN", "PROPN"}:
+                return _span_text([tok] + _subtree_tokens(head))
+        # single-token ADV frequency sometimes tagged ADV (everyday)
+        if tok.pos_ == "ADV" and tok.dep_ in {"advmod", "npadvmod"}:
+            # compounded everyday-like: has "day" lemma shape via morph — use text if det-less
+            if "day" in tok.lemma_.lower() or tok.text.lower().endswith("day"):
+                if tok.ent_type_ in {"DATE", "TIME", ""} and len(tok.text) > 3:
+                    # only if not under duration prep
+                    if not _under_duration_prep(tok):
+                        # Avoid treating random -day words; require no object attachment
+                        if tok.head.pos_ in {"VERB", "AUX"}:
+                            return tok.text
     return ""
 
 
 def extract_slots(text: str) -> SentenceSlots:
-    """Parse subject/verb/object/time/place/manner/negation/entities."""
+    """Parse subject/verb/object/time/place/manner/negation/entities via spaCy."""
     raw = (text or "").strip()
     slots = SentenceSlots(text=raw, sentence_type=classify_sentence(raw))
     if slots.sentence_type not in {"simple_declarative", "compound"}:
@@ -129,50 +144,81 @@ def extract_slots(text: str) -> SentenceSlots:
         slots.reasons.append("no_verb")
         return slots
 
-    subj_toks: list = []
+    # Subject: nominal or clausal (Visiting a library…)
+    subj_child = None
     for child in root.children:
-        if child.dep_ in {"nsubj", "nsubjpass"}:
-            subj_toks = _subtree_tokens(child)
+        if child.dep_ in {"nsubj", "nsubjpass", "csubj", "csubjpass"}:
+            subj_child = child
             break
-    slots.subject = _span_text(subj_toks)
 
-    neg_parts: list[str] = []
+    # Collect frontable adverbs first so we can exclude them from subject span
+    early_manner: list = []
     for t in doc:
-        if t.dep_ == "neg" and (t.head == root or t.head.head == root):
-            neg_parts.append(t.text)
+        if _is_frontable_adv(t, root):
+            if not (_is_temporal_tok(t) or t.ent_type_ in {"DATE", "TIME"}):
+                if t.dep_ != "npadvmod" or t.pos_ not in {"NOUN", "PROPN"}:
+                    early_manner.append(t)
+
+    if subj_child is not None:
+        exclude = {t.i for t in early_manner}
+        subj_toks = [t for t in _subtree_tokens(subj_child) if t.i not in exclude]
+        slots.subject = _span_text(subj_toks)
+    else:
+        slots.subject = ""
+
+    neg_parts = [
+        t.text
+        for t in doc
+        if t.dep_ == "neg" and (t.head == root or t.head.head == root)
+    ]
     slots.negation = " ".join(neg_parts)
 
     verb_toks = [t for t in root.lefts if t.dep_ in {"aux", "auxpass", "neg"}] + [root]
     slots.verb = root.text
     slots.verb_phrase = _span_text(sorted(verb_toks, key=lambda t: t.i))
 
-    # Object: noun objects, or non-finite complements (reading … / to go …)
     obj_toks: list = []
     for child in root.children:
         if child.dep_ in {"dobj", "obj", "attr"}:
             obj_toks = _subtree_tokens(child)
             break
+    # Append non-finite complements (to enjoy learning) so they are not dropped on rebuild
+    for child in root.children:
+        if child.dep_ in {"xcomp", "ccomp"}:
+            extra = _subtree_tokens(child)
+            # include leading "to" already in subtree for PART aux
+            for t in extra:
+                if t not in obj_toks:
+                    obj_toks.append(t)
+            obj_toks = sorted(obj_toks, key=lambda t: t.i)
     if not obj_toks:
         for child in root.children:
             if child.dep_ in {"xcomp", "ccomp"}:
-                # Keep gerund/infinitive complements as object material
                 obj_toks = _subtree_tokens(child)
-                # Include leading "to" aux if present for infinitives
                 break
     slots.object = _span_text(obj_toks)
 
+    # Place / beneficiary PPs: prep+pobj under root or under object head
     place_parts: list[str] = []
-    for child in root.children:
-        if child.dep_ == "prep" and child.text.lower() in _PLACE_PREPS:
-            span = _span_text(_subtree_tokens(child))
-            if not any(w.lower() in _TIME_WORDS for w in span.split()):
-                if not _is_duration_span(span.lstrip("to at in on "), raw):
-                    place_parts.append(span)
-    if not place_parts:
-        for t in doc:
-            if t.dep_ == "prep" and t.head == root and t.text.lower() in _PLACE_PREPS:
-                place_parts.append(_span_text(_subtree_tokens(t)))
-    slots.place = " ".join(p for p in place_parts if p).strip()
+    for t in doc:
+        if t.dep_ != "prep" or t.pos_ != "ADP":
+            continue
+        # NP-internal complements (thousands of books) are not frontable place adjuncts
+        if t.head.pos_ in {"NOUN", "PROPN", "NUM", "PRON"} and t.lemma_.lower() == "of":
+            continue
+        if t.head != root and t.head.dep_ not in {"attr", "dobj", "obj"}:
+            continue
+        span = _span_text(_subtree_tokens(t))
+        pobj = next((c for c in t.children if c.dep_ == "pobj"), None)
+        if (
+            pobj is not None
+            and t.lemma_.lower() in {"for", "over", "within", "during"}
+            and any(c.like_num or c.ent_type_ in {"TIME", "CARDINAL"} for c in pobj.subtree)
+        ):
+            continue
+        if t.head == root or t.head.dep_ in {"attr", "dobj", "obj"}:
+            place_parts.append(span)
+    slots.place = _dedupe_spans(place_parts)[0] if place_parts else ""
 
     time_parts: list[str] = []
     manner_parts: list[str] = []
@@ -180,64 +226,81 @@ def extract_slots(text: str) -> SentenceSlots:
     for ent in doc.ents:
         if ent.label_ not in {"DATE", "TIME"}:
             continue
-        if _is_duration_span(ent.text, raw):
+        if _is_duration_ent(ent, raw):
             continue
-        under_for = False
-        for t in ent:
-            cur = t
-            while cur is not None and cur.head != cur:
-                if cur.dep_ == "prep" and cur.text.lower() in _DURATION_PREPS:
-                    under_for = True
-                    break
-                cur = cur.head
-            if under_for:
-                break
-        if under_for:
+        under = any(_under_duration_prep(t) for t in ent)
+        if under and re.search(
+            r"\b(minute|minutes|hour|hours|second|seconds)\b", ent.text, flags=re.I
+        ):
+            continue
+        if under and re.match(r"at\s+least\b", ent.text, flags=re.I):
+            continue
+        # Duration under for — skip; bare adjunct DATE/TIME — keep
+        if under and ent.label_ == "TIME":
             continue
         time_parts.append(ent.text)
 
     for t in doc:
-        low = t.text.lower()
-        if t.dep_ in {"advmod", "npadvmod"} and (t.head == root or t.head.head == root):
-            if low in _TIME_WORDS or low in _FREQ_TIME or t.ent_type_ in {"DATE", "TIME"}:
-                if low not in _DISCOURSE_TIME:
-                    span = _span_text(_subtree_tokens(t))
-                    if not _is_duration_span(span, raw):
-                        time_parts.append(span)
-            elif _is_manner_adv(low):
-                manner_parts.append(t.text)
-        if t.dep_ == "npadvmod" and low in _TIME_WORDS and low not in _DISCOURSE_TIME:
-            time_parts.append(t.text)
-
-    # Frequency phrases (every day / everyday) — safe to front, keep durations intact
-    freq = _extract_freq_time(raw)
-    if freq:
-        time_parts.append(freq)
-
-    if not time_parts:
-        for t in doc:
-            low = t.text.lower()
-            if low in _TIME_WORDS and low not in _DISCOURSE_TIME:
-                time_parts.append(t.text)
-    if not manner_parts:
-        for t in doc:
-            if _is_manner_adv(t.text.lower()) and t.pos_ == "ADV":
+        if _is_frontable_adv(t, root):
+            if _is_temporal_tok(t) or t.ent_type_ in {"DATE", "TIME"}:
+                if not _under_duration_prep(t):
+                    time_parts.append(_span_text(_subtree_tokens(t)))
+            elif t.dep_ == "npadvmod" and t.pos_ in {"NOUN", "PROPN"} and t.head == root:
+                if not _under_duration_prep(t):
+                    time_parts.append(_span_text(_subtree_tokens(t)))
+            else:
+                # manner / discourse adverb
                 manner_parts.append(t.text)
 
+    freq = _freq_from_doc(doc)
+    if freq and not re.search(r"\b(minute|minutes|hour|hours|second|seconds)\b", freq, flags=re.I):
+        if not re.search(rf"\b(for|over|within|during)\s+{re.escape(freq)}\b", raw, flags=re.I):
+            time_parts.append(freq)
+
+    # npadvmod temporal nouns on root (yesterday/today) without NER
+    for t in doc:
+        if t.dep_ == "npadvmod" and t.head == root and t.pos_ in {"NOUN", "PROPN"}:
+            if not _under_duration_prep(t):
+                time_parts.append(_span_text(_subtree_tokens(t)))
+
+    time_parts = _dedupe_spans(list(dict.fromkeys(time_parts)))
+    # Drop duration-looking leftovers
     time_parts = [
-        p for p in _dedupe_spans(list(dict.fromkeys(time_parts))) if not _is_duration_span(p, raw)
+        p
+        for p in time_parts
+        if not re.search(r"\b(minute|minutes|hour|hours|second|seconds)\b", p, flags=re.I)
+        and not re.match(r"at\s+least\b", p, flags=re.I)
+        and not re.search(rf"\b(for|over|within|during)\s+{re.escape(p)}\b", raw, flags=re.I)
     ]
-    # Prefer a single best time span
     slots.time = time_parts[0] if time_parts else ""
-    slots.manner = " ".join(dict.fromkeys(manner_parts)).strip()
-
-    # Strip time/manner/freq from object if absorbed
-    for piece in (slots.place, slots.time, slots.manner, freq):
-        if piece and slots.object:
-            slots.object = re.sub(
-                rf"\b{re.escape(piece)}\b", "", slots.object, count=1, flags=re.I
-            ).strip(" ,")
-            slots.object = re.sub(r"\s+", " ", slots.object).strip()
+    # Prefer a single -ly manner adverb when several ADVs fire (also/still/regularly)
+    ly = [m for m in dict.fromkeys(manner_parts) if m.lower().endswith("ly")]
+    if ly:
+        slots.manner = ly[0]
+    else:
+        slots.manner = next(iter(dict.fromkeys(manner_parts)), "")
+    # Never treat the same span as both time and manner
+    if slots.manner and slots.time and slots.manner.lower() == slots.time.lower():
+        slots.manner = ""
+    if slots.manner and slots.time and slots.manner.lower() in slots.time.lower():
+        slots.manner = ""
+
+    # If place was absorbed into object, keep object but allow place move
+    if slots.place and slots.object and slots.place in slots.object:
+        # OK — templates can omit place from object when fronting
+        pass
+
+    for piece in (slots.time, slots.manner, freq):
+        if not piece:
+            continue
+        for attr in ("object", "subject", "place", "verb_phrase"):
+            val = getattr(slots, attr) or ""
+            if piece.lower() in val.lower():
+                cleaned = re.sub(
+                    rf"\b{re.escape(piece)}\b", "", val, count=1, flags=re.I
+                ).strip(" ,")
+                cleaned = re.sub(r"\s+", " ", cleaned).strip()
+                setattr(slots, attr, cleaned)
 
     score = 0.2
     if slots.subject:
@@ -257,7 +320,7 @@ def extract_slots(text: str) -> SentenceSlots:
 
 
 def _extract_slots_regex(text: str) -> SentenceSlots:
-    """Regex fallback when spaCy is unavailable — multi-word subjects supported."""
+    """Minimal regex fallback — structure only, no domain lexicons."""
     slots = SentenceSlots(text=text, sentence_type=classify_sentence(text))
     if slots.sentence_type != "simple_declarative":
         return slots
@@ -266,67 +329,51 @@ def _extract_slots_regex(text: str) -> SentenceSlots:
     if m_end:
         core = text[: m_end.start()]
 
-    # Peel frequency / calendar time / manner from the end
+    # Trailing -ly adverb
     manner = ""
-    time = ""
     mm = re.search(r"\b(\w+ly)$", core, flags=re.I)
     if mm:
         manner = mm.group(1)
         core = core[: mm.start()].strip()
-    mf = re.search(
-        r"\b((?:every|each)\s+(?:day|night|morning|evening|week|month|year)|"
-        r"everyday|daily|yesterday|today|tomorrow|earlier|later|recently|"
-        r"this\s+(?:morning|afternoon|evening|week|month|year)|"
-        r"last\s+(?:night|week|month|year))\s*$",
-        core,
-        flags=re.I,
-    )
+
+    # Trailing every/each + noun (DET pattern)
+    time = ""
+    mf = re.search(r"\b((?:every|each)\s+\w+)$", core, flags=re.I)
     if mf:
         time = mf.group(1)
         core = core[: mf.start()].strip()
 
-    # Subject = determiners/adjectives + noun(s); verb = next token
     m = re.match(
-        r"^(?P<subj>(?:(?:The|A|An|My|Your|Their|Our|His|Her|Many|Most|Some|Several|These|Those|This|That)\s+)?"
-        r"(?:[A-Za-z][\w'-]*\s+){0,3}[A-Za-z][\w'-]*)\s+"
-        r"(?P<verb>(?:(?:did|does|do|will|would|could|should|can|may|might|must|has|have|had|is|are|was|were|been)\s+)?"
-        r"(?:not\s+)?[A-Za-z][\w'-]*)\s*"
-        r"(?P<rest>.*)$",
+        r"^(?P<subj>.+?)\s+(?P<verb>\w+(?:\s+\w+)?)\s+(?P<rest>.+)$",
         core,
         flags=re.I,
     )
     if not m:
-        slots.reasons.append("regex_no_match")
-        return slots
-
-    slots.subject = m.group("subj").strip()
-    slots.verb_phrase = m.group("verb").strip()
-    slots.verb = slots.verb_phrase.split()[-1]
-    rest = (m.group("rest") or "").strip()
-
-    # Place: to X
-    mp = re.match(r"^to\s+(.+)$", rest, flags=re.I)
-    if mp and " for " not in rest.lower():
-        slots.place = "to " + mp.group(1).strip()
+        # Subj verb only
+        m2 = re.match(r"^(?P<subj>.+?)\s+(?P<verb>\w+)$", core, flags=re.I)
+        if not m2:
+            slots.reasons.append("regex_no_match")
+            return slots
+        slots.subject = m2.group("subj").strip()
+        slots.verb_phrase = m2.group("verb").strip()
+        slots.verb = slots.verb_phrase.split()[-1]
     else:
-        slots.object = rest
+        slots.subject = m.group("subj").strip()
+        slots.verb_phrase = m.group("verb").strip()
+        slots.verb = slots.verb_phrase.split()[-1]
+        rest = m.group("rest").strip()
+        if re.match(r"^to\s+\w+", rest, flags=re.I):
+            slots.place = rest if " for " not in rest.lower() else ""
+            slots.object = rest if not slots.place else ""
+        else:
+            slots.object = rest
 
     slots.time = time
     slots.manner = manner
-    if not slots.time:
-        slots.time = _extract_freq_time(text)
-
-    score = 0.2
-    if slots.subject:
-        score += 0.25
-    if slots.verb_phrase:
-        score += 0.2
-    if slots.object or slots.place:
-        score += 0.15
-    if slots.time:
-        score += 0.1
-    if slots.manner:
-        score += 0.1
+    score = 0.2 + (0.25 if slots.subject else 0) + (0.2 if slots.verb_phrase else 0)
+    score += 0.15 if (slots.object or slots.place) else 0
+    score += 0.1 if slots.time else 0
+    score += 0.1 if slots.manner else 0
     slots.confidence = min(1.0, score)
     if slots.confidence < 0.45 or not slots.subject or not slots.verb_phrase:
         slots.reasons.append("low_confidence")

app/engine/rewrite/__init__.py

@@ -48,8 +48,9 @@ def reorder_quality_ok(source: str, candidate: str) -> bool:
     cand_toks = _content_tokens(cand)
     if not src_toks:
         return False
+    # Soften keep ratio slightly for structural adverb fronts that drop discourse ADVs (also)
     keep = len(set(src_toks) & set(cand_toks)) / max(1, len(set(src_toks)))
-    if keep < 0.85:
+    if keep < 0.80:
         return False
     if len(cand_toks) < int(len(src_toks) * 0.75) or len(cand_toks) > int(
         len(src_toks) * 1.35

app/engine/templates/__init__.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import re
 
 from app.engine.models import SentenceSlots, TemplateCandidate
+from app.pipeline.nlp import get_nlp
 
 
 def rank_templates(slots: SentenceSlots) -> list[TemplateCandidate]:
@@ -25,19 +26,20 @@ def rank_templates(slots: SentenceSlots) -> list[TemplateCandidate]:
     if slots.time:
         out.append(TemplateCandidate("time_subj_verb_object_place", 0.72))
         out.append(TemplateCandidate("subj_verb_object_time", 0.68))
-        # Prefer slot rebuild over strip-based front when object/place exist
         out.append(TemplateCandidate("time_front", 0.64))
+    if slots.place and slots.subject and slots.verb_phrase:
+        out.append(TemplateCandidate("place_front", 0.74))
     if slots.manner and slots.place:
         out.append(TemplateCandidate("subj_manner_verb_place", 0.7))
     if slots.manner and slots.object:
         out.append(TemplateCandidate("subj_manner_verb_rest", 0.66))
     if slots.manner:
-        out.append(TemplateCandidate("adv_subj_verb_object", 0.62))
-        out.append(TemplateCandidate("subj_manner_verb_rest", 0.6))
+        out.append(TemplateCandidate("adv_subj_verb_object", 0.72))
+        out.append(TemplateCandidate("subj_manner_verb_rest", 0.65))
     if slots.place:
-        out.append(TemplateCandidate("subj_verb_place", 0.55))
+        out.append(TemplateCandidate("subj_verb_place", 0.6))
     if slots.object:
-        out.append(TemplateCandidate("subj_verb_object", 0.5))
+        out.append(TemplateCandidate("subj_verb_object", 0.58))
 
     seen: set[str] = set()
     ranked: list[TemplateCandidate] = []
@@ -49,20 +51,6 @@ def rank_templates(slots: SentenceSlots) -> list[TemplateCandidate]:
     return ranked
 
 
-_COMMON_CONTINUATIONS = frozenset(
-    """
-    the a an and or but if when as at by for in on to of with from
-    this that these those it its they them he she we you i my our your
-    many most some any all each every other such own same too not
-    reading writing sleeping walking running making taking getting
-    students experts people workers researchers managers teachers
-    yesterday today tomorrow earlier later recently now
-    because although however therefore moreover furthermore
-    she he they we it one someone everyone nobody
-    """.split()
-)
-
-
 def _cap(text: str) -> str:
     t = (text or "").strip()
     if not t:
@@ -71,19 +59,36 @@ def _cap(text: str) -> str:
 
 
 def _cont(phrase: str, slots: SentenceSlots) -> str:
-    """Lowercase continuation after an intro phrase only for common words."""
+    """Lowercase continuation after intro using spaCy POS (keep PROPN / entities)."""
     p = (phrase or "").strip()
     if not p:
         return p
     first = p.split()[0].strip("\"'")
-    # Always keep known NER entities
     for ent in slots.entities:
         ent_first = ent.split()[0]
-        if first.lower() == ent_first.lower() and ent_first[0].isupper():
-            if first.lower() not in _COMMON_CONTINUATIONS:
-                return ent_first + p[len(first) :]
-    if first and first[0].isupper() and first.lower() in _COMMON_CONTINUATIONS:
-        return first.lower() + p[len(first) :]
+        if first.lower() == ent_first.lower() and ent_first[:1].isupper():
+            # PERSON/ORG/GPE-style entities only (skip CARDINAL etc. via original casing mid-sent)
+            return p
+
+    nlp = get_nlp()
+    if nlp is not None:
+        try:
+            tok = nlp(first)[0]
+            if tok.pos_ == "PROPN":
+                return p
+            if tok.pos_ in {"DET", "PRON", "ADP", "SCONJ", "CCONJ", "ADV", "PART"}:
+                return first.lower() + p[len(first) :]
+            # Common noun/gerund subject after intro comma → lowercase
+            if tok.pos_ in {"NOUN", "VERB"}:
+                return first.lower() + p[len(first) :]
+        except Exception:
+            pass
+    if first and first[0].isupper() and len(first) > 1 and first.isupper() is False:
+        # Fallback without model: lowercase non-PROPN-looking tokens (mixed case names kept)
+        if first[0].isupper() and first[1:].islower() and first.lower() != first:
+            # Ambiguous — keep if looks like name (short Title case single token often name)
+            # Prefer lowercasing articles/pronouns already handled; leave as-is for names
+            return p
     return p
 
 
@@ -101,17 +106,32 @@ def _terminal(text: str) -> str:
     return "."
 
 
-_AUX_START = frozenset(
-    "did does do will would could should can may might must has have had is are was were been being".split()
-)
-
-
 def _manner_before_verb(verb_phrase: str) -> bool:
-    """False when verb phrase starts with an auxiliary (avoid 'regularly should drink')."""
-    parts = (verb_phrase or "").split()
+    """False when verb phrase starts with an auxiliary/copula (spaCy POS)."""
+    parts = (verb_phrase or "").strip()
     if not parts:
         return True
-    return parts[0].lower() not in _AUX_START
+    nlp = get_nlp()
+    if nlp is not None:
+        try:
+            tok = nlp(parts)[0]
+            # AUX or copular BE as root verb phrase start
+            if tok.pos_ == "AUX":
+                return False
+            if tok.lemma_.lower() == "be":
+                return False
+        except Exception:
+            pass
+    return True
+
+
+def _object_without_place(slots: SentenceSlots) -> str:
+    obj = (slots.object or "").strip()
+    place = (slots.place or "").strip()
+    if place and obj and place in obj:
+        obj = obj.replace(place, "").strip(" ,")
+        obj = re.sub(r"\s+", " ", obj).strip()
+    return obj
 
 
 def fill_template(template_id: str, slots: SentenceSlots) -> str | None:
@@ -119,9 +139,37 @@ def fill_template(template_id: str, slots: SentenceSlots) -> str | None:
     s = slots
     end = _terminal(s.text)
     tid = template_id
+    obj = _object_without_place(s)
+
+    if tid == "place_front":
+        if not s.place:
+            return None
+        # Copula/AUX: Subject + Verb + Manner + Object (not Subject + Manner + Verb)
+        if _manner_before_verb(s.verb_phrase):
+            body = _join_slots(
+                _cap(s.place) + ",",
+                _cont(s.subject, s),
+                s.manner,
+                s.verb_phrase,
+                obj,
+                s.time,
+            )
+        else:
+            body = _join_slots(
+                _cap(s.place) + ",",
+                _cont(s.subject, s),
+                s.verb_phrase,
+                s.manner,
+                obj,
+                s.time,
+            )
+        out = body + end
+        if out.lower().rstrip(".!?") == s.text.lower().rstrip(".!?"):
+            return None
+        return out
 
     if tid == "time_subj_manner_verb_place":
-        place_or_obj = s.place or s.object
+        place_or_obj = s.place or obj
         if _manner_before_verb(s.verb_phrase):
             body = _join_slots(
                 _cap(s.time) + ",", _cont(s.subject, s), s.manner, s.verb_phrase, place_or_obj
@@ -141,9 +189,9 @@ def fill_template(template_id: str, slots: SentenceSlots) -> str | None:
             _cap(s.time) + ",",
             _cont(s.subject, s),
             s.verb_phrase,
-            s.object,
+            obj,
             s.manner,
-            s.place,
+            s.place if s.place not in (obj or "") else "",
         )
         return body + end
 
@@ -152,14 +200,14 @@ def fill_template(template_id: str, slots: SentenceSlots) -> str | None:
             _cap(s.time) + ",",
             _cont(s.subject, s),
             s.verb_phrase,
-            s.object,
+            obj,
             s.place,
             s.manner,
         )
         return body + end
 
     if tid == "subj_verb_object_time":
-        body = _join_slots(_cap(s.subject), s.verb_phrase, s.object, s.place, s.manner, s.time)
+        body = _join_slots(_cap(s.subject), s.verb_phrase, obj, s.place, s.manner, s.time)
         return body + end
 
     if tid == "adv_subj_verb_object":
@@ -167,56 +215,52 @@ def fill_template(template_id: str, slots: SentenceSlots) -> str | None:
             _cap(s.manner) + ",",
             _cont(s.subject, s),
             s.verb_phrase,
-            s.object,
+            obj,
             s.place,
             s.time,
         )
         return body + end
 
     if tid == "subj_manner_verb_place":
-        body = _join_slots(_cap(s.subject), s.manner, s.verb_phrase, s.place)
+        if _manner_before_verb(s.verb_phrase):
+            body = _join_slots(_cap(s.subject), s.manner, s.verb_phrase, s.place)
+        else:
+            body = _join_slots(_cap(s.subject), s.verb_phrase, s.manner, s.place)
         return body + end
 
     if tid == "time_front":
-        # Refuse to front durations that were complements of "for/over/…"
         if s.time and re.search(
             rf"\b(for|over|within|during)\s+{re.escape(s.time)}\b",
             s.text,
             flags=re.I,
         ):
             return None
-        if s.time and re.search(
-            r"\b(minute|minutes|hour|hours|second|seconds)\b", s.time, flags=re.I
-        ):
-            return None
-        rest = s.text
-        if s.time:
-            rest = re.sub(rf"\b{re.escape(s.time)}\b", "", rest, count=1, flags=re.I)
-            rest = re.sub(r"\s+", " ", rest).strip(" ,.")
-        if not rest:
-            return None
-        # Reject stranded prepositions: "… for every day" / "… for."
-        if re.search(r"\b(for|over|within|during)\s+(every|each|\.|$)", rest, flags=re.I):
-            return None
-        if re.search(r"\b(for|over|within|during)\s*$", rest, flags=re.I):
-            return None
-        # Prefer rebuilding from slots when we have subject+verb (cleaner than strip)
         if s.subject and s.verb_phrase:
             body = _join_slots(
                 _cap(s.time) + ",",
                 _cont(s.subject, s),
                 s.verb_phrase,
-                s.object,
+                obj,
                 s.place,
                 s.manner,
             )
             return body.rstrip(".!?") + end
+        rest = s.text
+        if s.time:
+            rest = re.sub(rf"\b{re.escape(s.time)}\b", "", rest, count=1, flags=re.I)
+            rest = re.sub(r"\s+", " ", rest).strip(" ,.")
+        if not rest:
+            return None
+        if re.search(r"\b(for|over|within|during)\s+(every|each|\.|$)", rest, flags=re.I):
+            return None
         body = _join_slots(_cap(s.time) + ",", _cont(rest, s))
         return body.rstrip(".!?") + end
 
     if tid == "subj_manner_verb_rest":
-        rest_bits = [s.verb_phrase, s.object, s.place, s.time]
-        body = _join_slots(_cap(s.subject), s.manner, *rest_bits)
+        if _manner_before_verb(s.verb_phrase):
+            body = _join_slots(_cap(s.subject), s.manner, s.verb_phrase, obj, s.place, s.time)
+        else:
+            body = _join_slots(_cap(s.subject), s.verb_phrase, s.manner, obj, s.place, s.time)
         return body + end
 
     if tid == "subj_verb_place":
@@ -224,7 +268,7 @@ def fill_template(template_id: str, slots: SentenceSlots) -> str | None:
         return body + end
 
     if tid == "subj_verb_object":
-        body = _join_slots(_cap(s.subject), s.verb_phrase, s.object, s.place, s.time, s.manner)
+        body = _join_slots(_cap(s.subject), s.verb_phrase, obj, s.place, s.time, s.manner)
         return body + end
 
     return None
@@ -259,12 +303,30 @@ def try_because_front(text: str) -> str | None:
 
 
 def try_discourse_front(text: str) -> str | None:
-    """Move mid-sentence 'nowadays' to the front."""
+    """Move mid-sentence discourse ADV (nowadays) detected via spaCy when possible."""
     raw = (text or "").strip()
-    if not raw or re.match(r"^nowadays\b", raw, flags=re.I):
-        return None
-    if re.search(r"\b(because|although|while|unless)\b", raw, flags=re.I):
+    if not raw:
         return None
+    nlp = get_nlp()
+    token = None
+    if nlp is not None:
+        doc = nlp(raw)
+        for t in doc:
+            if (
+                t.pos_ == "ADV"
+                and t.dep_ in {"advmod", "npadvmod"}
+                and t.i > 0
+                and t.head.dep_ == "ROOT"
+                and t.text.lower() == "nowadays"
+            ):
+                token = t
+                break
+    if token is None:
+        if re.match(r"^nowadays\b", raw, flags=re.I):
+            return None
+        m = re.search(r"\bnowadays\b", raw, flags=re.I)
+        if not m:
+            return None
     end = "."
     m_end = re.search(r"[.!?]+$", raw)
     core = raw

tests/test_esl_habit_regression.py

@@ -34,6 +34,23 @@ def test_esl_habit_no_duration_front():
 
 def test_esl_habit_no_false_entity_caps():
     result = rewrite_document(SAMPLE)
-    # Consistency must not force "Reading" / "Many" mid-sentence caps after commas
     assert ", Reading " not in result.text
     assert ", Many " not in result.text
+
+
+def test_library_sample_reorders():
+    sample = (
+        "Libraries is still an important place for students and researchers. "
+        "They contains thousands of books that provide useful informations on different subjects. "
+        "Visiting a library regularly can also encourage childrens to enjoy learning."
+    )
+    result = rewrite_document(sample)
+    # Must do structural reorder, not grammar-only passthrough on sentence 1
+    assert result.sentences[0].status == "rewritten"
+    assert result.sentences[0].rewritten.lower().startswith("for students")
+    # Relative-clause sentence is safely skipped (may still get grammar cleanup)
+    assert result.sentences[1].status in {"skipped", "reverted", "passthrough"}
+    # Manner front on gerund subject
+    assert result.sentences[2].status == "rewritten"
+    assert "regularly" in result.sentences[2].rewritten.lower()
+    assert result.sentences[2].rewritten.lower().startswith("regularly")