Upload 100 files

app/pipeline/candidate_validator.py

@@ -63,16 +63,78 @@ def _extract_propers(text: str) -> set[str]:
     return found
 
 
+_STOP = frozenset(
+    """
+    a an the and or but if in on at to for of as by with from into over after
+    before about than then so too very more most such that this these those
+    it its they them their we our you your he she his her is are was were be
+    been being have has had do does did will would can could should may might
+    not no nor also just only own same other another each every both few many
+    much some any all because while when where how what which who whom
+    """.split()
+)
+
+
+def _content_tokens(text: str) -> set[str]:
+    """Lowercased content words (length ≥4) used for coverage checks."""
+    toks = re.findall(r"[a-zA-Z']+", text.lower())
+    out: set[str] = set()
+    for t in toks:
+        t = t.strip("'")
+        if len(t) < 4 or t in _STOP:
+            continue
+        # Light stem: drop trailing s/ed/ing for overlap
+        stem = t
+        if t.endswith("ing") and len(t) > 5:
+            stem = t[:-3]
+        elif t.endswith("ed") and len(t) > 4:
+            stem = t[:-2]
+        elif t.endswith("es") and len(t) > 4:
+            stem = t[:-2]
+        elif t.endswith("s") and len(t) > 4:
+            stem = t[:-1]
+        out.add(stem)
+    return out
+
+
+def _sentence_count(text: str) -> int:
+    parts = re.split(r"[.!?]+", (text or "").strip())
+    return len([p for p in parts if p.strip()])
+
+
 def _length_ok(original: str, candidate: str) -> bool:
     ow = max(1, len(original.split()))
     cw = len(candidate.split())
-    if cw < max(3, ow // 3):
+    # Reject half-dropped paraphrases (FLAN often keeps only the last sentence).
+    if ow <= 80:
+        min_words = max(3, int(ow * 0.72))
+    else:
+        min_words = max(3, int(ow * 0.55))
+    if cw < min_words:
         return False
     if cw > int(ow * 2.2) + 8:
         return False
     return True
 
 
+def _coverage_ok(original: str, candidate: str) -> bool:
+    """Reject dropped sentences / severe content loss; allow normal paraphrases."""
+    o_sents = _sentence_count(original)
+    c_sents = _sentence_count(candidate)
+    # Short multi-sentence passages must keep every sentence (anti half-drop).
+    if 2 <= o_sents <= 5 and c_sents < o_sents:
+        return False
+
+    o_toks = _content_tokens(original)
+    if len(o_toks) >= 5:
+        c_toks = _content_tokens(candidate)
+        overlap = len(o_toks & c_toks) / max(1, len(o_toks))
+        # Only fail on severe stem loss (paraphrases often rewrite many content words).
+        if overlap < 0.32:
+            return False
+    return True
+
+
 def _meaning_score(original: str, candidate: str) -> float | None:
     """MiniLM cosine if available; None if model missing."""
     try:
@@ -112,6 +174,9 @@ def validate_candidate(
     if not _length_ok(o, c):
         reasons.append("length")
 
+    if not _coverage_ok(o, c):
+        reasons.append("coverage")
+
     surf = _surface_sim(o, c)
     if surf >= max_surface:
         reasons.append("too_similar")

app/pipeline/generative.py

@@ -35,6 +35,7 @@ _failed = False
 _backend: str | None = None
 
 _SHARED_RULES = (
+    "Keep ALL sentences and claims from the paragraph — do not drop or merge away ideas.\n"
     "Use different wording and vary sentence structure — do not copy phrases "
     "from the input almost verbatim.\n"
     "Keep every claim and the same polarity. Do not flip negatives or antonyms.\n"

app/pipeline/orchestrator.py

@@ -146,7 +146,7 @@ def _accept_generative(source: str, draft: str) -> bool:
 
     def _hard_fail(reasons: list[str]) -> bool:
         for r in reasons:
-            if r == "polarity" or r in {"numbers", "quotes"}:
+            if r in {"polarity", "numbers", "quotes", "length", "coverage"}:
                 return True
             if r.startswith("meaning") or r.startswith("entity"):
                 return True

scripts/test_quality_pipeline.py

@@ -31,6 +31,21 @@ def test_polarity() -> None:
     print("polarity OK")
 
 
+def test_validator_rejects_truncated() -> None:
+    orig = (
+        "Nowadays many people are living an unhealthy life because they don't have enough time. "
+        "Eating fast foods is becoming very common and people don't realize how much it affects their health."
+    )
+    truncated = (
+        "Fast food is becoming very common because many people don't realize "
+        "how much it affects their health."
+    )
+    v = validate_candidate(orig, truncated, min_meaning=0.5)
+    assert not v.ok
+    assert "length" in v.reasons or "coverage" in v.reasons
+    print("validator rejects truncated OK:", v.reasons)
+
+
 def test_validator_rejects_flips() -> None:
     orig = (
         "Nowadays many people are living an unhealthy life because they don't have enough time. "
@@ -151,6 +166,7 @@ def test_classical_fallback() -> None:
 if __name__ == "__main__":
     test_polarity()
     test_validator_rejects_flips()
+    test_validator_rejects_truncated()
     test_validator_rejects_near_copy()
     test_ranker_rejects_near_copy_prefers_paraphrase()
     test_ranker_prefers_faithful()