Upload 107 files

app/pipeline/generative.py

@@ -37,7 +37,7 @@ _backend: str | None = None
 _backend_kind: str | None = None  # seq2seq | causal
 
 _SYSTEM_REWRITE = (
-    "You rewrite ONE English sentence with clearly different wording and structure. "
+    "You rewrite ONE English sentence with clearly DIFFERENT WORDS. "
     "Output exactly one sentence — no second sentence, no preamble. "
     "Keep EVERY claim and the same polarity. "
     "Do not flip negatives or antonyms. "
@@ -46,7 +46,8 @@ _SYSTEM_REWRITE = (
     "Do not add reasons, examples, education, awareness, routines, "
     "workouts, diets, or new facts. "
     "Do not summarize or drop content. "
-    "Do NOT copy the input almost verbatim — change words and word order. "
+    "Do NOT rearrange the same words. Do NOT copy almost verbatim. "
+    "Replace verbs, adverbs, and phrases with natural synonyms. "
     "Return only the rewritten sentence."
 )
 
@@ -56,7 +57,7 @@ _SHARED_RULES = (
     "Do not add reasons, education, awareness, routines, workouts, diets, "
     "or extra clauses.\n"
     "Keep multiword terms intact (e.g. fast food, junk food, physical activity).\n"
-    "MUST use different wording and structure — never return a near-copy.\n"
+    "MUST use different wording — never rearrange the same words only.\n"
     "Keep polarity. Preserve names, numbers, dates, and quotations. Correct grammar.\n"
     "Return only the rewritten text."
 )
@@ -81,22 +82,22 @@ _TONE_PROMPTS: dict[str, str] = {
         "Paraphrase:"
     ),
     "Neutral": (
-        "Paraphrase this text in clear natural English with clearly different wording.\n"
+        "Paraphrase this text in clear natural English using different words.\n"
         f"{_SHARED_RULES}\n"
         "Keep polarity (e.g. unhealthy stays unhealthy; "
         "don't realize stays a lack of awareness — without adding 'education').\n"
-        "Do not return a grammar-only edit of the same sentence.\n\n"
+        "Do not return a grammar-only edit or a shuffle of the same words.\n\n"
         "Text:\n{text}\n\n"
         "Paraphrase:"
     ),
 }
 
 _RETRY_PROMPT = (
-    "Rewrite this as ONE sentence with clearly different wording and structure "
+    "Rewrite this as ONE sentence using DIFFERENT WORDS (not a word shuffle) "
     "but the exact same meaning and polarity. Do not add or remove facts. "
     "Keep fixed phrases like 'fast food'. "
     "Do not invent workouts, diets, or routines. Do not add a second sentence. "
-    "Do not return a near-copy or grammar-only edit. "
+    "Do not return a near-copy, grammar-only edit, or rearranged same words. "
     "Tone: {tone}.\n\n"
     "Text:\n{text}\n\n"
     "Rewrite:"

app/pipeline/orchestrator.py

@@ -177,7 +177,7 @@ def _paraphrase_hard_ok(source_unit: str, trial: str) -> bool:
 def _is_true_rewrite(source_unit: str, candidate: str) -> bool:
     """True rewrite = different wording, not grammar-fix or clause shuffle.
 
-    Pure reorder / template edits keep almost the same content stems and must fail.
+    Pure reorder of the same words must fail (e.g. Ram/school shuffle example).
     """
     src = (source_unit or "").strip()
     cand = (candidate or "").strip()
@@ -185,18 +185,32 @@ def _is_true_rewrite(source_unit: str, candidate: str) -> bool:
         return False
     if cand.lower().rstrip(".!?") == src.lower().rstrip(".!?"):
         return False
+    # Same content-word bag → reorder only (not a complete rewrite)
+    src_bag = _content_word_bag(src)
+    cand_bag = _content_word_bag(cand)
+    if src_bag and src_bag == cand_bag:
+        return False
     if not _paraphrase_hard_ok(src, cand):
         return False
     sim = _similarity_ratio(src, cand)
     jacc = _stem_jaccard(src, cand)
-    # Same bag of content words → reorder/template only (user complaint)
-    if jacc >= 0.92:
+    # Near-identical stem set → not enough wording change
+    if jacc >= 0.88:
         return False
-    if sim > 0.90:
+    if sim > 0.88:
         return False
     return True
 
 
+def _content_word_bag(text: str) -> frozenset[str]:
+    """Lowercased content words (len≥4), for detecting same-word shuffles."""
+    return frozenset(
+        w.strip("'")
+        for w in re.findall(r"[a-zA-Z']+", (text or "").lower())
+        if len(w.strip("'")) >= 4
+    )
+
+
 def _safe_paraphrase_trial(source_unit: str, trial: str) -> tuple[bool, float]:
     """Return (is_true_rewrite, surface_sim)."""
     t = (trial or "").strip()
@@ -212,7 +226,12 @@ def _rewrite_score(source_unit: str, candidate: str) -> float:
     """Higher = more different while still usable (lower sim + lower stem overlap)."""
     sim = _similarity_ratio(source_unit, candidate)
     jacc = _stem_jaccard(source_unit, candidate)
-    return (1.0 - sim) + 0.75 * (1.0 - jacc)
+    bag_bonus = (
+        0.15
+        if _content_word_bag(source_unit) != _content_word_bag(candidate)
+        else 0.0
+    )
+    return (1.0 - sim) + 0.85 * (1.0 - jacc) + bag_bonus
 
 
 # Phrase-level meaning-preserving rewrites (change wording, not just order).
@@ -254,6 +273,18 @@ _PHRASE_REWRITES: tuple[tuple[re.Pattern[str], str], ...] = (
     (re.compile(r"\bkeeps our body fit\b", re.I), "helps the body stay fit"),
     (re.compile(r"\bfind it harder to\b", re.I), "struggle more to"),
     (re.compile(r"\binstead of\b", re.I), "rather than"),
+    # General complete-rewrite lexical swaps (not reorder)
+    (re.compile(r"\bwent to\b", re.I), "headed to"),
+    (re.compile(r"\bgo to\b", re.I), "head to"),
+    (re.compile(r"\bgoes to\b", re.I), "heads to"),
+    (re.compile(r"\bhappily\b", re.I), "cheerfully"),
+    (re.compile(r"\bquickly\b", re.I), "in a hurry"),
+    (re.compile(r"\bslowly\b", re.I), "at a slow pace"),
+    (re.compile(r"\byesterday\b", re.I), "the day before"),
+    (re.compile(r"\btomorrow\b", re.I), "the next day"),
+    (re.compile(r"\bstarted\b", re.I), "began"),
+    (re.compile(r"\bfinished\b", re.I), "completed"),
+    (re.compile(r"\bhelped\b", re.I), "assisted"),
 )
 
 
@@ -303,9 +334,10 @@ def _esl_structural_variants(text: str) -> list[str]:
     if m:
         topic = m.group(1).strip().rstrip(".!?")
         if topic:
-            out.append(f"{_cap_first(topic)} also matters a lot{end}")
-            out.append(f"Getting {_lower_first(topic)} is also important{end}")
-            out.append(f"{_cap_first(topic)} deserves attention too{end}")
+            # Keep topic head "thing" so topic_anchor passes; change other wording
+            out.append(f"{_cap_first(topic)} remains an important thing{end}")
+            out.append(f"{_cap_first(topic)} is still an important thing{end}")
+            out.append(f"A further important thing is {_lower_first(topic)}{end}")
 
     m = re.match(r"^Without enough\s+(\w+),\s*(.+)$", core, flags=re.I)
     if m:
@@ -382,8 +414,8 @@ _SAFE_SURFACE_SWAPS: tuple[tuple[re.Pattern[str], str], ...] = (
     (re.compile(r"\bhelp communities\b", re.I), "support communities"),
 )
 
-_VISIBLE_TARGET_SIM = 0.85
-_LM_SHIP_MAX_SIM = 0.88
+_VISIBLE_TARGET_SIM = 0.82
+_LM_SHIP_MAX_SIM = 0.85
 
 
 def _surface_humanize_variants(text: str, rng: random.Random) -> list[str]:
@@ -891,7 +923,11 @@ def _hybrid_rewrite(
                     by_para.setdefault(unit.paragraph_index, []).append(cand_u)
                 continue
             safe = _safe_classical_unit(src_u, cand_u)
-            if safe == src_u:
+            if strength >= 1 and len(src_u.split()) >= 4:
+                safe = _require_visible_rewrite(
+                    src_u, safe, tone, rng, target_sim=_VISIBLE_TARGET_SIM
+                )
+            if _similarity_ratio(src_u, safe) > 0.92:
                 stats.reverted_source += 1
             else:
                 stats.classical_kept += 1

scripts/test_phase_a_guards.py

@@ -152,6 +152,35 @@ def test_exercise_tv_must_not_be_reorder_only() -> None:
     print("  stats", stats)
 
 
+def test_complete_wording_rewrite_not_shuffle() -> None:
+    """Complete rewrite = new words; same-word shuffle must not count."""
+    import random
+
+    from app.pipeline.orchestrator import (
+        _content_word_bag,
+        _is_true_rewrite,
+        _require_visible_rewrite,
+        _similarity_ratio,
+    )
+
+    src = "Ram went to school yesterday happily."
+    shuffle = "Yesterday, Ram happily went to school."
+    assert not _is_true_rewrite(src, shuffle), "word shuffle must fail"
+    assert _content_word_bag(src) == _content_word_bag(shuffle)
+
+    out = _require_visible_rewrite(
+        src, src, "Neutral", random.Random(5), target_sim=0.82
+    )
+    assert _is_true_rewrite(src, out), out
+    assert _content_word_bag(src) != _content_word_bag(out)
+    assert "ram" in out.lower()
+    assert "school" in out.lower()
+    # Should use different wording (headed/cheerfully/day before), not shuffle
+    assert "yesterday, ram happily went" not in out.lower()
+    assert _similarity_ratio(src, out) < 0.90
+    print("Ram complete wording rewrite OK:", out)
+
+
 def test_rejects_hollow_ons_bills() -> None:
     src = (
         "People should save money for emergencies instead of spending every paycheck."
@@ -464,6 +493,7 @@ if __name__ == "__main__":
     test_skip_short_and_light()
     test_sleep_style_must_visibly_rewrite()
     test_exercise_tv_must_not_be_reorder_only()
+    test_complete_wording_rewrite_not_shuffle()
     test_rejects_hollow_ons_bills()
     test_assembly_preserves_unit_count()
     test_live_bug_patterns_rejected()