Upload 105 files

app/pipeline/candidate_validator.py

@@ -367,6 +367,27 @@ def _invention_ok(original: str, candidate: str) -> bool:
     if "program" in novel or "programme" in novel:
         if novel & {"exercise", "fitness", "gym", "workout", "training"}:
             return False
+    # Invented "learn/learning to <verb>" when verb not in source (e.g. learning to read)
+    for m in re.finditer(
+        r"\b(?:learn(?:ing)?|study(?:ing)?)\s+to\s+([a-zA-Z]+)\b",
+        (candidate or ""),
+        flags=re.I,
+    ):
+        verb = m.group(1).lower()
+        stem = verb
+        if stem.endswith("ing") and len(stem) > 5:
+            stem = stem[:-3]
+        elif stem.endswith("ed") and len(stem) > 4:
+            stem = stem[:-2]
+        elif stem.endswith("s") and len(stem) > 3:
+            stem = stem[:-1]
+        if stem not in o_toks and verb not in o_toks:
+            return False
+    # Invented "check with …" meeting frame not in source
+    if re.search(r"\bcheck with\b", candidate or "", flags=re.I) and not re.search(
+        r"\bcheck with\b", original or "", flags=re.I
+    ):
+        return False
     return True
 
 

app/pipeline/meaning_safety.py

@@ -164,6 +164,40 @@ def _lemma_still_negated(text: str, lemma: str) -> bool:
     return False
 
 
+def _comparison_sides(text: str) -> tuple[set[str], set[str]] | None:
+    """Extract content stems on each side of a less/more…than comparison."""
+    m = re.search(
+        r"^(.{3,80}?)\b(?:less|more|fewer|greater|better|worse)\b"
+        r".{0,100}?\bthan\s+(.{3,80}?)(?:[.!?]|$)",
+        (text or "").strip(),
+        flags=re.I | re.S,
+    )
+    if not m:
+        return None
+    left = _stems_in_text(m.group(1))
+    right = _stems_in_text(m.group(2))
+    if not left or not right:
+        return None
+    return left, right
+
+
+def _comparison_order_ok(original: str, candidate: str) -> bool:
+    """Reject swapped comparison subjects (Electric…than petrol → Petrol…than electric)."""
+    sides_o = _comparison_sides(original)
+    sides_c = _comparison_sides(candidate)
+    if not sides_o or not sides_c:
+        return True
+    lo, ro = sides_o
+    lc, rc = sides_c
+    lo_rc = len(lo & rc) / max(1, len(lo))
+    lo_lc = len(lo & lc) / max(1, len(lo))
+    ro_lc = len(ro & lc) / max(1, len(ro))
+    # Original left aligns with candidate right, and original right with candidate left
+    if lo_rc >= 0.45 and ro_lc >= 0.45 and lo_lc < 0.45:
+        return False
+    return True
+
+
 def polarity_safe(original: str, candidate: str) -> bool:
     """Reject candidates that flip antonyms or strip/add negation on key verbs."""
     if not original.strip() or not candidate.strip():
@@ -220,4 +254,7 @@ def polarity_safe(original: str, candidate: str) -> bool:
         if not _NEG_EQUIV.search(c):
             return False
 
+    if not _comparison_order_ok(original, candidate):
+        return False
+
     return True

app/pipeline/orchestrator.py

@@ -185,6 +185,24 @@ def _sentence_count_simple(text: str) -> int:
     return len([p for p in parts if p.strip()])
 
 
+def _clamp_to_source_shape(source_unit: str, candidate: str) -> str:
+    """Keep LM output to the same sentence count as the source unit (1→1)."""
+    c = (candidate or "").strip()
+    if not c:
+        return ""
+    o_n = _sentence_count_simple(source_unit)
+    c_n = _sentence_count_simple(c)
+    if o_n == 1 and c_n != 1:
+        # Take first sentence only; if still multi or empty, reject
+        first = re.split(r"(?<=[.!?])\s+", c, maxsplit=1)[0].strip()
+        if _sentence_count_simple(first) == 1 and first:
+            return first
+        return ""
+    if o_n >= 1 and c_n < o_n:
+        return ""
+    return c
+
+
 def _classical_paragraph(paragraph: str, tone: str, strength: int, rng: random.Random) -> str:
     """Offline fallback: structure + curated lexicon (not the primary quality path)."""
     styled = apply_tone_style(paragraph, tone, strength, rng)
@@ -357,6 +375,13 @@ def _accept_lm_unit(source_unit: str, candidate: str, *, max_sim: float = 0.90)
     """Strict gate for a single LM-rewritten sentence."""
     if not candidate or not candidate.strip():
         return False, ["empty"]
+    # Hard 1:1 sentence shape before other gates
+    o_n = _sentence_count_simple(source_unit)
+    c_n = _sentence_count_simple(candidate)
+    if o_n == 1 and c_n != 1:
+        return False, ["shape"]
+    if o_n >= 1 and c_n < o_n:
+        return False, ["shape"]
     v = validate_candidate(
         source_unit, candidate, min_meaning=0.80, max_surface=max_sim
     )
@@ -463,9 +488,7 @@ def _hybrid_rewrite(
             max_sim=0.90,
         )
         gen_clean = tidy(correct_text(gen_raw) if gen_raw else "")
-        # Reject multi-sentence LM output for a single source unit
-        if _sentence_count_simple(gen_clean) > 1 and _sentence_count_simple(src_u) == 1:
-            gen_clean = ""
+        gen_clean = _clamp_to_source_shape(src_u, gen_clean)
         ok, reasons = _accept_lm_unit(src_u, gen_clean, max_sim=0.90)
 
         if ok:
@@ -482,11 +505,7 @@ def _hybrid_rewrite(
                     max_sim=0.88,
                 )
                 gen_clean2 = tidy(correct_text(gen_raw2) if gen_raw2 else "")
-                if (
-                    _sentence_count_simple(gen_clean2) > 1
-                    and _sentence_count_simple(src_u) == 1
-                ):
-                    gen_clean2 = ""
+                gen_clean2 = _clamp_to_source_shape(src_u, gen_clean2)
                 ok2, reasons2 = _accept_lm_unit(src_u, gen_clean2, max_sim=0.88)
                 if ok2:
                     stats.gen_accepted += 1
@@ -512,9 +531,13 @@ def _hybrid_rewrite(
 
         if not (chosen or "").strip():
             chosen = src_u
-        # One source unit → one sentence max in the slot
-        if _sentence_count_simple(chosen) > 1 and _sentence_count_simple(src_u) == 1:
-            chosen = _light_revert_polish(src_u, tone, rng, strength=1)
+        chosen = _clamp_to_source_shape(src_u, chosen) or src_u
+        # Never let an accepted unit expand into multiple sentences
+        if _sentence_count_simple(chosen) != _sentence_count_simple(src_u):
+            if _sentence_count_simple(src_u) == 1:
+                chosen = _light_revert_polish(src_u, tone, rng, strength=1)
+            else:
+                chosen = src_u
         outputs.append(_ensure_unit_terminal(chosen))
 
     if len(outputs) != len(source_units):
@@ -527,8 +550,21 @@ def _hybrid_rewrite(
             outputs.append(_ensure_unit_terminal(source_units[len(outputs)][1]))
         outputs = outputs[: len(source_units)]
 
+    # Per-slot sanity: empty/too-short slots → source unit
+    fixed_outs: list[str] = []
+    for (_pi, src_u), out in zip(source_units, outputs):
+        piece = (out or "").strip()
+        if (
+            not piece
+            or _sentence_count_simple(piece) < _sentence_count_simple(src_u)
+            or len(piece.split()) < max(3, int(len(src_u.split()) * 0.55))
+        ):
+            piece = _ensure_unit_terminal(src_u)
+            stats.bump_reason("unit_restore")
+        fixed_outs.append(_ensure_unit_terminal(piece))
+    outputs = fixed_outs
+
     # Assemble ONLY from per-unit strings — no document-level grammar here
-    # (document LT was observed dropping sentences, e.g. Reading sample).
     text = _assemble_hybrid_units(source_units, outputs)
     scrubbed = tidy(_light_post_gen(text, tone))
     if len(iter_source_units(scrubbed)) < len(source_units):
@@ -540,9 +576,15 @@ def _hybrid_rewrite(
         text = _assemble_hybrid_units(source_units, outputs)
     else:
         text = scrubbed
-    # Final hard guarantee
+    # Final hard guarantee — never ship fewer units than source
     if len(iter_source_units(text)) < len(source_units):
+        logger.warning("Final hybrid text lost units; restoring assembled outputs")
         text = _assemble_hybrid_units(source_units, outputs)
+    if len(iter_source_units(text)) < len(source_units):
+        text = _assemble_hybrid_units(
+            source_units,
+            [_ensure_unit_terminal(u) for _i, u in source_units],
+        )
     return text, stats, used_gen
 
 
@@ -760,6 +802,26 @@ def rewrite_text(
             else:
                 rewritten = tidy(cleaned_out)
 
+    # Nuclear hybrid safety: never return fewer sentence units than the source
+    if mode == "hybrid" and hybrid_stats and hybrid_stats.units:
+        n_src = hybrid_stats.units
+        if len(iter_source_units(rewritten)) < n_src:
+            logger.warning(
+                "Post-pipeline unit loss (%s→%s); restoring from grammar source units",
+                n_src,
+                len(iter_source_units(rewritten)),
+            )
+            src_units = iter_source_units(source)
+            rewritten = _assemble_hybrid_units(
+                src_units,
+                [
+                    _ensure_unit_terminal(
+                        _light_revert_polish(u, tone, rng, strength=1)
+                    )
+                    for _i, u in src_units
+                ],
+            )
+
     ratio = _similarity_ratio(source, rewritten)
     changed = rewritten.strip() != original.strip()
 

scripts/test_phase_a_guards.py

@@ -270,6 +270,64 @@ def test_assemble_preserves_two_units() -> None:
     print("assemble two units OK:", text)
 
 
+def test_rejects_comparison_swap() -> None:
+    src = "Electric cars produce less air pollution than petrol cars."
+    bad = "Petrol cars make less air pollution than electric cars."
+    from app.pipeline.meaning_safety import polarity_safe
+
+    assert not polarity_safe(src, bad)
+    v = validate_candidate(src, bad, min_meaning=0.5)
+    assert not v.ok
+    assert "polarity" in v.reasons
+    ok, reasons = _accept_lm_unit(src, bad)
+    assert not ok
+    print("comparison swap reject OK:", v.reasons, reasons)
+
+
+def test_rejects_learning_to_read_invention() -> None:
+    src = "Learning a second language opens more career opportunities."
+    bad = "Learning to read a second language opens many career opportunities."
+    v = validate_candidate(src, bad, min_meaning=0.5)
+    assert not v.ok
+    assert "invention" in v.reasons
+    print("learning-to-read invention reject OK:", v.reasons)
+
+
+def test_rejects_check_with_teachers_invention() -> None:
+    src = (
+        "Teachers should check progress regularly instead of grading only "
+        "the final report."
+    )
+    bad = (
+        "Teachers should check with teachers about progress instead of "
+        "grading the final report."
+    )
+    v = validate_candidate(src, bad, min_meaning=0.5)
+    assert not v.ok
+    assert "invention" in v.reasons
+    print("check-with-teachers invention reject OK:", v.reasons)
+
+
+def test_clamp_rejects_multi_sentence() -> None:
+    from app.pipeline.orchestrator import (
+        _accept_lm_unit,
+        _clamp_to_source_shape,
+        _sentence_count_simple,
+    )
+
+    src = "Learning a second language opens more career opportunities."
+    bad = (
+        "Learning a second language opens more career opportunities. "
+        "Travel becomes easier too."
+    )
+    clamped = _clamp_to_source_shape(src, bad)
+    assert _sentence_count_simple(clamped) == 1
+    ok, reasons = _accept_lm_unit(src, bad)
+    assert not ok
+    assert "shape" in reasons
+    print("multi-sentence clamp/reject OK:", reasons)
+
+
 if __name__ == "__main__":
     test_rejects_multi_sentence_invention()
     test_rejects_two_sentence_for_one()
@@ -288,4 +346,8 @@ if __name__ == "__main__":
     test_rejects_garbage_dontstet()
     test_rejects_recycle_negation_flip()
     test_assemble_preserves_two_units()
+    test_rejects_comparison_swap()
+    test_rejects_learning_to_read_invention()
+    test_rejects_check_with_teachers_invention()
+    test_clamp_rejects_multi_sentence()
     print("\nALL PHASE A TESTS PASSED")