Upload 107 files

app/pipeline/candidate_validator.py

@@ -200,6 +200,7 @@ _WEAK_TOPIC_HEADS = frozenset(
     local young loud hard free quiet small large great important different
     second first major main recent modern social early late daily
     without within enough several various certain further additional
+    unfortunately sadly however therefore moreover furthermore actually
     """.split()
 )
 

app/pipeline/orchestrator.py

@@ -121,78 +121,172 @@ def _ensure_unit_terminal(text: str) -> str:
     return s
 
 
-def _safe_paraphrase_trial(source_unit: str, trial: str) -> tuple[bool, float]:
-    """Return (ok_for_visible_use, surface_sim). Soft-only validation failures allowed."""
+def _unit_stems(text: str) -> set[str]:
+    """Content stems shared with the validator (for lexical-novelty checks)."""
+    from app.pipeline.candidate_validator import _content_tokens
+
+    return _content_tokens(text or "")
+
+
+def _stem_keep_ratio(source: str, candidate: str) -> float:
+    src = _unit_stems(source)
+    if not src:
+        return 1.0
+    return len(src & _unit_stems(candidate)) / len(src)
+
+
+def _stem_jaccard(source: str, candidate: str) -> float:
+    a, b = _unit_stems(source), _unit_stems(candidate)
+    if not a and not b:
+        return 1.0
+    return len(a & b) / max(1, len(a | b))
+
+
+def _paraphrase_hard_ok(source_unit: str, trial: str) -> bool:
+    """Meaning safety for forced rewrites.
+
+    Allows real synonym/phrase changes (relaxed key_content/collocation) but still
+    blocks polarity flips, invention, topic swaps, etc.
+    """
     t = (trial or "").strip()
     if not t:
-        return False, 1.0
+        return False
     t = _clamp_to_source_shape(source_unit, t) or t
-    t = _ensure_unit_terminal(t)
-    v = validate_candidate(source_unit, t, min_meaning=0.70, max_surface=0.995)
-    soft = {"too_similar", "identical", "near_copy", "too_divergent"}
-    hard_keys = [
-        r.split(":")[0] for r in v.reasons if r.split(":")[0] not in soft
-    ]
+    v = validate_candidate(source_unit, t, min_meaning=0.62, max_surface=0.995)
+    # key_content/coverage/collocation/drift are too strict for synonym paraphrases;
+    # enforce a softer stem floor instead.
+    soft = {
+        "too_similar",
+        "identical",
+        "near_copy",
+        "too_divergent",
+        "key_content",
+        "coverage",
+        "drift",
+        "grammar_worse",
+        "collocation",
+    }
+    hard_keys = [r.split(":")[0] for r in v.reasons if r.split(":")[0] not in soft]
     if _hard_fail(hard_keys):
-        return False, 1.0
-    return True, _similarity_ratio(source_unit, t)
+        return False
+    if _stem_keep_ratio(source_unit, t) < 0.30:
+        return False
+    return True
 
 
-_SAFE_SURFACE_SWAPS: tuple[tuple[re.Pattern[str], str], ...] = (
-    (re.compile(r"\bIn addition,\b", re.I), "Also,"),
-    (re.compile(r"\bAdditionally,\b", re.I), "Also,"),
-    (re.compile(r"\bHowever,\b", re.I), "Still,"),
-    (re.compile(r"\bTherefore,\b", re.I), "So,"),
-    (re.compile(r"\bDue to the fact that\b", re.I), "Because"),
-    (re.compile(r"\bin order to\b", re.I), "to"),
-    (re.compile(r"\ba large number of\b", re.I), "many"),
-    (re.compile(r"\bMany\b"), "A lot of"),
-    (re.compile(r"\bA lot of\b", re.I), "Many"),
-    (re.compile(r"\bis able to\b", re.I), "can"),
-    (re.compile(r"\bare able to\b", re.I), "can"),
-    (re.compile(r"\bPeople should\b"), "People need to"),
-    (re.compile(r"\bpeople should\b"), "people need to"),
-    (re.compile(r"\binstead of\b", re.I), "rather than"),
-    (re.compile(r"\brather than\b", re.I), "instead of"),
-    (re.compile(r"\bbecause of\b", re.I), "due to"),
-    (re.compile(r"\bfor example\b", re.I), "for instance"),
-    (re.compile(r"\bfor instance\b", re.I), "for example"),
-    (re.compile(r"\bthink that\b", re.I), "think"),
-    (re.compile(r"\bbelieve that\b", re.I), "believe"),
-    (re.compile(r"\bsay that\b", re.I), "say"),
-    (re.compile(r"\bis too\b", re.I), "is really too"),
-    (re.compile(r"\bare too\b", re.I), "are really too"),
-    (re.compile(r"\bbetter for (.+?) than\b", re.I), r"better for \1 compared to"),
-    (re.compile(r"\bbetter for (.+?) compared to\b", re.I), r"better for \1 than"),
-    (re.compile(r"\bby giving\b", re.I), "by offering"),
-    (re.compile(r"\bhelp communities\b", re.I), "support communities"),
-    (re.compile(r"\bwastes their\b", re.I), "uses up their"),
-    (re.compile(r"\bfind it harder to\b", re.I), "have a harder time"),
-    (re.compile(r"\bhave a harder time\b", re.I), "find it harder to"),
-    (re.compile(r"\bit is important to\b", re.I), "people need to"),
-    (re.compile(r"\bThere are many\b"), "Plenty of"),
-)
+def _is_true_rewrite(source_unit: str, candidate: str) -> bool:
+    """True rewrite = different wording, not grammar-fix or clause shuffle.
 
-_VISIBLE_TARGET_SIM = 0.85
-_LM_SHIP_MAX_SIM = 0.88
+    Pure reorder / template edits keep almost the same content stems and must fail.
+    """
+    src = (source_unit or "").strip()
+    cand = (candidate or "").strip()
+    if not src or not cand:
+        return False
+    if cand.lower().rstrip(".!?") == src.lower().rstrip(".!?"):
+        return False
+    if not _paraphrase_hard_ok(src, cand):
+        return False
+    sim = _similarity_ratio(src, cand)
+    jacc = _stem_jaccard(src, cand)
+    # Same bag of content words → reorder/template only (user complaint)
+    if jacc >= 0.92:
+        return False
+    if sim > 0.90:
+        return False
+    return True
 
 
-def _cap_first(text: str) -> str:
-    t = (text or "").strip()
+def _safe_paraphrase_trial(source_unit: str, trial: str) -> tuple[bool, float]:
+    """Return (is_true_rewrite, surface_sim)."""
+    t = (trial or "").strip()
     if not t:
-        return t
-    return t[0].upper() + t[1:]
+        return False, 1.0
+    t = _ensure_unit_terminal(_clamp_to_source_shape(source_unit, t) or t)
+    if not _is_true_rewrite(source_unit, t):
+        return False, 1.0
+    return True, _similarity_ratio(source_unit, t)
 
 
-def _lower_first(text: str) -> str:
-    t = (text or "").strip()
-    if not t:
-        return t
-    return t[0].lower() + t[1:]
+def _rewrite_score(source_unit: str, candidate: str) -> float:
+    """Higher = more different while still usable (lower sim + lower stem overlap)."""
+    sim = _similarity_ratio(source_unit, candidate)
+    jacc = _stem_jaccard(source_unit, candidate)
+    return (1.0 - sim) + 0.75 * (1.0 - jacc)
+
+
+# Phrase-level meaning-preserving rewrites (change wording, not just order).
+# Prefer swaps that keep enough claim stems to stay meaning-safe.
+_PHRASE_REWRITES: tuple[tuple[re.Pattern[str], str], ...] = (
+    (re.compile(r"\bget(?:s)?\s+diseases\b", re.I), "get sick"),
+    (re.compile(r"\bgetting\s+diseases\b", re.I), "getting sick"),
+    (
+        re.compile(r"\bless chance to gets?\s+diseases\b", re.I),
+        "less chance to get sick",
+    ),
+    (
+        re.compile(r"\bless chance of getting\s+diseases\b", re.I),
+        "less chance of getting sick",
+    ),
+    (re.compile(r"\bwatching television\b", re.I), "watching TV"),
+    (re.compile(r"\bdoing physical activities\b", re.I), "working out"),
+    (re.compile(r"\bphysical activities\b", re.I), "real exercise"),
+    (re.compile(r"\bexercise regularly\b", re.I), "work out often"),
+    (re.compile(r"\bexercises regularly\b", re.I), "works out often"),
+    (re.compile(r"\ba person exercise\b", re.I), "a person works out"),
+    (re.compile(r"\ba person exercises\b", re.I), "a person works out"),
+    (re.compile(r"\bprefers watching\b", re.I), "would rather watch"),
+    (re.compile(r"\bprefer watching\b", re.I), "would rather watch"),
+    (re.compile(r"\bmany peoples\b", re.I), "a lot of people"),
+    (re.compile(r"\bmany people\b", re.I), "a lot of people"),
+    (re.compile(r"\bUnfortunately,\s*", re.I), "Sadly, "),
+    (
+        re.compile(
+            r"\bis one of the most important\s+\w+\s+that keeps our body fit\b",
+            re.I,
+        ),
+        "is a key way to keep the body fit",
+    ),
+    (
+        re.compile(r"\bis one of the most important\b", re.I),
+        "is a key",
+    ),
+    (re.compile(r"\bkeeps our body fit\b", re.I), "helps the body stay fit"),
+    (re.compile(r"\bfind it harder to\b", re.I), "struggle more to"),
+    (re.compile(r"\binstead of\b", re.I), "rather than"),
+)
+
+
+def _apply_phrase_rewrites(text: str, *, max_swaps: int = 4) -> list[str]:
+    """Build increasingly rewritten phrase variants."""
+    out: list[str] = []
+    base = (text or "").strip()
+    if not base:
+        return out
+    # Single-swap variants
+    for pat, repl in _PHRASE_REWRITES:
+        if not pat.search(base):
+            continue
+        cand = pat.sub(repl, base, count=1)
+        if cand.strip() and cand.strip() != base:
+            out.append(cand.strip())
+    # Stacked multi-swap
+    stacked = base
+    n = 0
+    for pat, repl in _PHRASE_REWRITES:
+        if n >= max_swaps:
+            break
+        if pat.search(stacked):
+            nxt = pat.sub(repl, stacked, count=1)
+            if nxt != stacked:
+                stacked = nxt
+                n += 1
+                out.append(stacked.strip())
+    return out
 
 
 def _esl_structural_variants(text: str) -> list[str]:
-    """High-value ESL / claim rewrites that change wording while keeping meaning."""
+    """Lexical ESL rewrites (must change wording — not mere reorder)."""
     out: list[str] = []
     base = (text or "").strip()
     if not base:
@@ -201,7 +295,6 @@ def _esl_structural_variants(text: str) -> list[str]:
     end = end_m.group(0) if end_m else "."
     core = base[: end_m.start()] if end_m else base
 
-    # Another/One important thing is/are X → keep important/thing stems + topic
     m = re.match(
         r"^(?:Another|One|An)\s+important\s+thing\s+(?:is|are)\s+(.+)$",
         core,
@@ -210,26 +303,18 @@ def _esl_structural_variants(text: str) -> list[str]:
     if m:
         topic = m.group(1).strip().rstrip(".!?")
         if topic:
-            out.append(f"{_cap_first(topic)} is another important thing{end}")
-            out.append(f"{_cap_first(topic)} is also an important thing{end}")
-            out.append(f"One more important thing is {_lower_first(topic)}{end}")
+            out.append(f"{_cap_first(topic)} also matters a lot{end}")
+            out.append(f"Getting {_lower_first(topic)} is also important{end}")
+            out.append(f"{_cap_first(topic)} deserves attention too{end}")
 
-    # Without enough X, Y → reorder / rephrase while keeping enough+X(+Y)
     m = re.match(r"^Without enough\s+(\w+),\s*(.+)$", core, flags=re.I)
     if m:
         noun, rest = m.group(1), m.group(2).strip()
         if noun and rest:
-            # Clause reorder keeps all stems (best for incomplete fragments)
-            out.append(f"{_cap_first(rest)} without enough {noun}{end}")
-            out.append(
-                f"When there is not enough {noun}, {_lower_first(rest)}{end}"
-            )
-            out.append(f"With not enough {noun}, {_lower_first(rest)}{end}")
-            out.append(
-                f"If there is not enough {noun}, {_lower_first(rest)}{end}"
-            )
+            out.append(f"When {noun} runs short, {_lower_first(rest)}{end}")
+            out.append(f"If {noun} is lacking, {_lower_first(rest)}{end}")
+            out.append(f"Too little {noun} leaves {_lower_first(rest)}{end}")
 
-    # X is one of the most important Y
     m = re.match(
         r"^(.+?)\s+is one of the most important\s+(.+)$",
         core,
@@ -238,50 +323,80 @@ def _esl_structural_variants(text: str) -> list[str]:
     if m:
         subj, rest = m.group(1).strip(), m.group(2).strip()
         if subj and rest:
-            out.append(f"{_cap_first(subj)} is a very important part of {rest}{end}")
-            out.append(
-                f"Among the most important {rest}, {_lower_first(subj)} stands out{end}"
-            )
+            out.append(f"{_cap_first(subj)} is a key part of {rest}{end}")
+            out.append(f"{_cap_first(subj)} matters a lot for {rest}{end}")
 
-    # It is important that/to …
     m = re.match(r"^It is important (?:that|to)\s+(.+)$", core, flags=re.I)
     if m:
         rest = m.group(1).strip()
         if rest:
             out.append(f"People need to {_lower_first(rest)}{end}")
-            out.append(f"{_cap_first(rest)} is important{end}")
+            out.append(f"{_cap_first(rest)} really matters{end}")
+
+    # If a person X, they will Y → People who X will Y
+    m = re.match(
+        r"^(?:If\s+)?a person\s+(.+?),\s*they will\s+(.+)$",
+        core,
+        flags=re.I,
+    )
+    if m:
+        act, result = m.group(1).strip(), m.group(2).strip()
+        out.append(f"People who {act} will {result}{end}")
+        out.append(f"Anyone who {act} will {result}{end}")
 
     return out
 
 
+def _cap_first(text: str) -> str:
+    t = (text or "").strip()
+    if not t:
+        return t
+    return t[0].upper() + t[1:]
+
+
+def _lower_first(text: str) -> str:
+    t = (text or "").strip()
+    if not t:
+        return t
+    return t[0].lower() + t[1:]
+
+
+_SAFE_SURFACE_SWAPS: tuple[tuple[re.Pattern[str], str], ...] = (
+    (re.compile(r"\bIn addition,\b", re.I), "Also,"),
+    (re.compile(r"\bAdditionally,\b", re.I), "Also,"),
+    (re.compile(r"\bHowever,\b", re.I), "Still,"),
+    (re.compile(r"\bTherefore,\b", re.I), "So,"),
+    (re.compile(r"\bDue to the fact that\b", re.I), "Because"),
+    (re.compile(r"\bin order to\b", re.I), "to"),
+    (re.compile(r"\ba large number of\b", re.I), "many"),
+    (re.compile(r"\bMany\b"), "A lot of"),
+    (re.compile(r"\bis able to\b", re.I), "can"),
+    (re.compile(r"\bare able to\b", re.I), "can"),
+    (re.compile(r"\bPeople should\b"), "People need to"),
+    (re.compile(r"\bpeople should\b"), "people need to"),
+    (re.compile(r"\bbecause of\b", re.I), "due to"),
+    (re.compile(r"\bfor example\b", re.I), "for instance"),
+    (re.compile(r"\bthink that\b", re.I), "think"),
+    (re.compile(r"\bbelieve that\b", re.I), "believe"),
+    (re.compile(r"\bby giving\b", re.I), "by offering"),
+    (re.compile(r"\bhelp communities\b", re.I), "support communities"),
+)
+
+_VISIBLE_TARGET_SIM = 0.85
+_LM_SHIP_MAX_SIM = 0.88
+
+
 def _surface_humanize_variants(text: str, rng: random.Random) -> list[str]:
-    """Stem-preserving phrase/style variants — prefer these over aggressive lexicon thrash."""
+    """Phrase + light surface variants that change wording."""
     out: list[str] = []
     base = (text or "").strip()
     if not base:
         return out
 
+    out.extend(_apply_phrase_rewrites(base))
     out.extend(_esl_structural_variants(base))
 
-    # Intensifier insert before adjectives after "too/very/so"
-    m = re.search(r"\b(too|very|so)\s+([a-zA-Z]{4,})\b", base)
-    if m and "really" not in base.lower():
-        out.append(base[: m.start()] + f"really {m.group(0)}" + base[m.end() :])
-
-    # Front "Instead of X, Y" from "Y instead of X"
-    m = re.search(
-        r"^(?P<main>.+?)\s+instead of\s+(?P<alt>.+?)(?P<end>[.!?]?)$",
-        base,
-        flags=re.I,
-    )
-    if m:
-        main = m.group("main").rstrip(" ,")
-        alt = m.group("alt").rstrip(" .!?")
-        end = m.group("end") or "."
-        main_l = _lower_first(main)
-        out.append(f"Instead of {alt}, {main_l}{end}")
-
-    # Because-clause fronting: "X because Y" → "Because Y, X"
+    # Because-clause fronting only when combined with a phrase rewrite later
     m = re.match(
         r"^(?P<main>.+?)\s+because\s+(?P<reason>.+?)(?P<end>[.!?])?$",
         base,
@@ -301,18 +416,10 @@ def _surface_humanize_variants(text: str, rng: random.Random) -> list[str]:
         cand = pat.sub(repl, base, count=1)
         if cand.strip() and cand.strip() != base:
             out.append(cand.strip())
-    stacked = base
-    applied = 0
-    for pat, repl in swaps:
-        if applied >= 2:
-            break
-        if pat.search(stacked):
-            nxt = pat.sub(repl, stacked, count=1)
-            if nxt != stacked:
-                stacked = nxt
-                applied += 1
-    if stacked.strip() and stacked.strip() != base:
-        out.append(stacked.strip())
+    # Stack phrase rewrites on surface swaps
+    for seed in list(out[:8]):
+        out.extend(_apply_phrase_rewrites(seed))
+        out.extend(_esl_structural_variants(seed))
     return out
 
 
@@ -324,7 +431,7 @@ def _visible_unit_paraphrase(
     strength: int = 2,
     target_sim: float = _VISIBLE_TARGET_SIM,
 ) -> str:
-    """Force visible wording change on revert — surface + structure + light lexicon."""
+    """Produce a true lexical rewrite — not grammar-only or clause shuffle."""
     base = scrub_phrases(source_unit)
     base = apply_tone_contractions(base, tone)
     base = tidy(correct_text(base) if base else "")
@@ -344,26 +451,33 @@ def _visible_unit_paraphrase(
         structured = base
 
     trials: list[str] = []
-    trials.extend(_esl_structural_variants(base))
-    trials.extend(_surface_humanize_variants(base, rng))
-    trials.extend(_surface_humanize_variants(structured, rng))
+    for seed in (base, structured):
+        trials.extend(_apply_phrase_rewrites(seed))
+        trials.extend(_esl_structural_variants(seed))
+        trials.extend(_surface_humanize_variants(seed, rng))
     if structured.strip() and structured.strip() != base.strip():
         trials.append(structured)
 
-    for syn_src, syn_str in ((structured, 1), (base, 1), (structured, 2), (base, 2)):
+    # Synonym passes — allow stronger lexicon; safety filter keeps meaning
+    for syn_src, syn_str, force in (
+        (structured, 2, True),
+        (base, 2, True),
+        (structured, 1, True),
+        (base, 1, False),
+    ):
         try:
             cand = rewrite_sentence_synonyms(
                 syn_src,
                 syn_str,
                 rng,
                 tone=tone,
-                force_all_lexicon=False,
+                force_all_lexicon=force,
             )
             cand = apply_tone_contractions(cand, tone)
             cand = tidy(correct_text(cand) if cand else "")
             if cand.strip():
                 trials.append(cand)
-                trials.extend(_surface_humanize_variants(cand, rng))
+                trials.extend(_apply_phrase_rewrites(cand))
                 trials.extend(_esl_structural_variants(cand))
         except Exception:
             continue
@@ -378,7 +492,7 @@ def _visible_unit_paraphrase(
         uniq.append(t.strip())
 
     best = _ensure_unit_terminal(source_unit)
-    best_sim = 1.0
+    best_score = -1.0
     for trial in uniq:
         ok, sim = _safe_paraphrase_trial(source_unit, trial)
         if not ok:
@@ -386,13 +500,26 @@ def _visible_unit_paraphrase(
         t = _ensure_unit_terminal(
             _clamp_to_source_shape(source_unit, trial.strip()) or trial.strip()
         )
-        if t.lower().rstrip(".!?") == source_unit.lower().rstrip(".!?"):
-            continue
-        if sim < best_sim:
+        score = _rewrite_score(source_unit, t)
+        if score > best_score or (
+            abs(score - best_score) < 1e-9 and sim < _similarity_ratio(source_unit, best)
+        ):
             best = t
-            best_sim = sim
-            if sim <= target_sim:
-                break
+            best_score = score
+
+    # Final phrase polish on the winner — push more wording change when safe
+    if best_score >= 0:
+        for polished in _apply_phrase_rewrites(best, max_swaps=4):
+            if not _is_true_rewrite(source_unit, polished):
+                continue
+            t = _ensure_unit_terminal(
+                _clamp_to_source_shape(source_unit, polished.strip())
+                or polished.strip()
+            )
+            score = _rewrite_score(source_unit, t)
+            if score > best_score:
+                best = t
+                best_score = score
     return best
 
 
@@ -403,7 +530,7 @@ def _light_revert_polish(
     *,
     strength: int = 0,
 ) -> str:
-    """Backward-compatible name — now runs visible classical paraphrase."""
+    """Backward-compatible name — now runs true lexical paraphrase."""
     return _visible_unit_paraphrase(
         source_unit,
         tone,
@@ -422,49 +549,34 @@ def _require_visible_rewrite(
     target_sim: float = _VISIBLE_TARGET_SIM,
     min_words: int = 4,
 ) -> str:
-    """Never ship grammar-only near-copies when a safe different sentence exists."""
+    """Never ship grammar-only / reorder-only when a true rewrite exists."""
     src = (source_unit or "").strip()
     cand = (candidate or src).strip() or src
     if len(src.split()) < min_words:
         return _ensure_unit_terminal(cand)
 
-    cand_sim = _similarity_ratio(src, cand)
-    if (
-        cand_sim <= target_sim
-        and cand.lower().rstrip(".!?") != src.lower().rstrip(".!?")
-    ):
-        ok, _ = _safe_paraphrase_trial(src, cand)
-        if ok:
-            return _ensure_unit_terminal(cand)
+    if _is_true_rewrite(src, cand) and _similarity_ratio(src, cand) <= target_sim:
+        return _ensure_unit_terminal(cand)
 
-    # Prefer classical paraphrase of source (not of a hollow LM near-copy)
+    # Classical true rewrite from source (ignore weak LM near-copies)
     trials = [
         _visible_unit_paraphrase(src, tone, rng, strength=2, target_sim=target_sim),
-        _visible_unit_paraphrase(src, tone, rng, strength=2, target_sim=0.80),
+        _visible_unit_paraphrase(src, tone, rng, strength=2, target_sim=0.78),
     ]
-    if cand.lower().rstrip(".!?") != src.lower().rstrip(".!?"):
-        trials.append(
-            _visible_unit_paraphrase(
-                cand, tone, rng, strength=2, target_sim=target_sim
-            )
-        )
-
     best = _ensure_unit_terminal(cand)
-    best_sim = cand_sim
+    best_score = -1.0
+    if _is_true_rewrite(src, best):
+        best_score = _rewrite_score(src, best)
     for trial in trials:
-        ok, sim = _safe_paraphrase_trial(src, trial)
-        if not ok:
+        if not _is_true_rewrite(src, trial):
             continue
         t = _ensure_unit_terminal(
             _clamp_to_source_shape(src, trial.strip()) or trial.strip()
         )
-        if t.lower().rstrip(".!?") == src.lower().rstrip(".!?"):
-            continue
-        if sim < best_sim:
+        score = _rewrite_score(src, t)
+        if score > best_score:
             best = t
-            best_sim = sim
-            if sim <= target_sim:
-                break
+            best_score = score
     return best
 
 
@@ -476,7 +588,7 @@ def _boost_if_near_copy(
     *,
     threshold: float = _VISIBLE_TARGET_SIM,
 ) -> str:
-    """If a kept unit is still nearly identical, force classical paraphrase."""
+    """If a kept unit is still nearly identical, force a true rewrite."""
     return _require_visible_rewrite(
         source_unit,
         candidate,
@@ -701,30 +813,38 @@ def _looks_needs_lm(text: str, strength: int) -> bool:
 def _accept_lm_unit(
     source_unit: str, candidate: str, *, max_sim: float = _LM_SHIP_MAX_SIM
 ) -> tuple[bool, list[str]]:
-    """Strict gate for a single LM-rewritten sentence."""
+    """Strict gate for a single LM-rewritten sentence — must be a true rewrite."""
     if not candidate or not candidate.strip():
         return False, ["empty"]
-    # Hard 1:1 sentence shape before other gates
     o_n = _sentence_count_simple(source_unit)
     c_n = _sentence_count_simple(candidate)
     if o_n == 1 and c_n != 1:
         return False, ["shape"]
     if o_n >= 1 and c_n < o_n:
         return False, ["shape"]
+    sim = _similarity_ratio(source_unit, candidate)
+    if sim >= max_sim or sim >= _LM_SHIP_MAX_SIM:
+        return False, ["near_copy"]
+    # Must be a true lexical rewrite (not grammar/reorder)
+    if not _is_true_rewrite(source_unit, candidate):
+        return False, ["near_copy"]
     v = validate_candidate(
-        source_unit, candidate, min_meaning=0.80, max_surface=max_sim
+        source_unit, candidate, min_meaning=0.72, max_surface=max_sim
     )
-    reasons = list(v.reasons)
-    sim = _similarity_ratio(source_unit, candidate)
-    if sim >= max_sim:
-        reasons.append("near_copy")
-    # Even if validate passed soft gates, do not ship near-copies as LM success
-    if sim >= _LM_SHIP_MAX_SIM:
-        if "near_copy" not in reasons:
-            reasons.append("near_copy")
-        return False, reasons or ["near_copy"]
-    if not v.ok or "near_copy" in reasons:
-        return False, reasons or ["reject"]
+    soft = {
+        "too_similar",
+        "identical",
+        "near_copy",
+        "too_divergent",
+        "key_content",
+        "coverage",
+        "drift",
+        "grammar_worse",
+        "collocation",
+    }
+    hard_keys = [r.split(":")[0] for r in v.reasons if r.split(":")[0] not in soft]
+    if _hard_fail(hard_keys):
+        return False, list(v.reasons) or ["reject"]
     return True, []
 
 

scripts/test_phase_a_guards.py

@@ -63,29 +63,30 @@ def test_skip_short_and_light() -> None:
 def test_sleep_style_must_visibly_rewrite() -> None:
     """User expectation: not grammar-only; wording must clearly change."""
     import random
+    from unittest.mock import patch
 
     from app.pipeline.orchestrator import (
+        _is_true_rewrite,
         _require_visible_rewrite,
         _similarity_ratio,
+        _stem_jaccard,
     )
 
     rng = random.Random(7)
     src = "Another important thing is sleeping."
     out = _require_visible_rewrite(src, src, "Neutral", rng, target_sim=0.85)
-    assert out.lower().rstrip(".!?") != src.lower().rstrip(".!?")
+    assert _is_true_rewrite(src, out), out
     assert _similarity_ratio(src, out) < 0.90
+    assert _stem_jaccard(src, out) < 0.92
     assert "sleep" in out.lower()
     print("sleep opener rewrite OK:", out)
 
     src2 = "Without enough sleep, the brain."
     out2 = _require_visible_rewrite(src2, src2, "Neutral", rng, target_sim=0.85)
-    assert out2.lower().rstrip(".!?") != src2.lower().rstrip(".!?")
+    assert _is_true_rewrite(src2, out2), out2
     assert "sleep" in out2.lower()
     print("without-enough rewrite OK:", out2)
 
-    # Full paste via hybrid with near-copy LM → must still visibly rewrite
-    from unittest.mock import patch
-
     raw = "Another important thing are sleeping. Without enough sleep, the brain"
     g = correct_text(raw)
     with patch(
@@ -99,13 +100,56 @@ def test_sleep_style_must_visibly_rewrite() -> None:
         )
     assert _similarity_ratio(g, out) < 0.90
     assert "sleep" in out.lower()
+    # Must not be grammar-only near-copy of the classic paste
+    assert "another important thing is sleeping" not in out.lower()
+    print("sleep document hybrid rewrite OK:", out)
+
+
+def test_exercise_tv_must_not_be_reorder_only() -> None:
+    """User complaint sample: must change wording, keep regularly/claims."""
+    import random
+    from unittest.mock import patch
+
+    from app.pipeline.orchestrator import (
+        _hybrid_rewrite,
+        _is_true_rewrite,
+        _similarity_ratio,
+    )
+
+    raw = (
+        "Exercise is one of the most important activity that keeps our body fit. "
+        "If a person exercise regularly, they will have less chance to gets diseases. "
+        "Unfortunately, many peoples prefers watching television instead of doing "
+        "physical activities."
+    )
+    g = correct_text(raw)
+    rng = random.Random(11)
+    with patch(
+        "app.pipeline.orchestrator.rewrite_unit",
+        side_effect=lambda u, **_k: u,
+    ):
+        out, stats, _ = _hybrid_rewrite(g, "Neutral", 1, rng, can_generate=True)
     low = out.lower()
-    assert "another important thing is sleeping" not in low
+    assert _similarity_ratio(g, out) < 0.90
+    # Not the weak template / reorder-only patterns users rejected
+    assert "very important part of activities that keeps" not in low
     assert not (
-        low.startswith("another important thing")
-        and "without enough sleep, the brain" in low
+        "instead of doing physical activities, unfortunately" in low
+        and "watching television" in low
     )
-    print("sleep document hybrid rewrite OK:", out)
+    # Claims should survive in some form
+    assert "tv" in low or "television" in low
+    assert "fit" in low or "exercise" in low or "work out" in low or "workout" in low
+    # Spot-check units are true rewrites vs grammar source units
+    from app.pipeline.alignment import iter_source_units
+
+    src_units = [u for _i, u in iter_source_units(g)]
+    out_units = [u for _i, u in iter_source_units(out)]
+    assert len(out_units) == len(src_units)
+    true_n = sum(1 for s, o in zip(src_units, out_units) if _is_true_rewrite(s, o))
+    assert true_n >= max(1, len(src_units) - 1), (true_n, out)
+    print("exercise/TV rewrite OK:", out)
+    print("  stats", stats)
 
 
 def test_rejects_hollow_ons_bills() -> None:
@@ -419,6 +463,7 @@ if __name__ == "__main__":
     test_rejects_two_sentence_for_one()
     test_skip_short_and_light()
     test_sleep_style_must_visibly_rewrite()
+    test_exercise_tv_must_not_be_reorder_only()
     test_rejects_hollow_ons_bills()
     test_assembly_preserves_unit_count()
     test_live_bug_patterns_rejected()