Upload 102 files

README.md

@@ -110,7 +110,8 @@ Add Supabase secrets, then add the Space URL to Supabase Auth redirect URLs.
 
 ## Notes
 
-- Core rewrite engine is classical NLP (spaCy + WordNet + rules).
+- With **ML polish**: generative rewrite is primary (`flan-t5-base` by default); classical lexicon is fallback. See [docs/ML_POLISH.md](docs/ML_POLISH.md).
+- Without ML polish: classical NLP (spaCy + WordNet + rules + grammar).
 - With Supabase enabled: **guest preview** on the homepage → sign up for Free → Pro later.
 - Admin: manage `profiles` / `plans` in the Supabase Table Editor.
 - Stripe can set `plan_id` later; see setup doc.

app/config.py

@@ -63,20 +63,29 @@ LANGUAGE_TOOL_ENABLED = _lt_flag not in {"0", "false", "no", "off"} and bool(LAN
 MINILM_MODEL = (
     os.environ.get("MINILM_MODEL") or "sentence-transformers/all-MiniLM-L6-v2"
 ).strip()
-# Tiny seq2seq for ML polish draft (CPU). Override to try other FLAN/T5 checkpoints.
+# Seq2seq rewrite model. Prefer base over small for quality; override via env.
+# Larger instruction models (7B+) need GPU — set GENERATIVE_MODEL explicitly.
 GENERATIVE_MODEL = (
-    os.environ.get("GENERATIVE_MODEL") or "google/flan-t5-small"
+    os.environ.get("GENERATIVE_MODEL") or "google/flan-t5-base"
+).strip()
+# 0 = rewrite entire document (no paragraph cap). Positive = safety cap.
+_gen_max_raw = (
+    os.environ.get("GENERATIVE_MAX_PARAGRAPHS")
+    or os.environ.get("GENERATIVE_MAX_SENTENCES")
+    or "0"
 ).strip()
-# 0 = paraphrase the entire uploaded document (no sentence cap).
-# Positive value = optional safety cap only (e.g. 200). Hard-clamped to 5000.
-_gen_max_raw = (os.environ.get("GENERATIVE_MAX_SENTENCES") or "0").strip()
 try:
     _gen_max_n = int(_gen_max_raw)
 except ValueError:
     _gen_max_n = 0
-GENERATIVE_MAX_SENTENCES = max(0, min(_gen_max_n, 5000))
+GENERATIVE_MAX_PARAGRAPHS = max(0, min(_gen_max_n, 5000))
+# Back-compat alias
+GENERATIVE_MAX_SENTENCES = GENERATIVE_MAX_PARAGRAPHS
 _gen_flag = (os.environ.get("GENERATIVE_POLISH_ENABLED") or "true").strip().lower()
 GENERATIVE_POLISH_ENABLED = _gen_flag not in {"0", "false", "no", "off"}
+# When true and generative model loads, prefer gen path whenever ml_polish is requested
+_gen_primary = (os.environ.get("GENERATIVE_PRIMARY") or "true").strip().lower()
+GENERATIVE_PRIMARY = _gen_primary not in {"0", "false", "no", "off"}
 _ml_flag = (os.environ.get("ML_POLISH_ENABLED") or "true").strip().lower()
 ML_POLISH_AVAILABLE_DEFAULT = _ml_flag not in {"0", "false", "no", "off"}
 _ml_warm = (os.environ.get("ML_POLISH_WARM") or "false").strip().lower()
@@ -88,3 +97,7 @@ _gfi = (os.environ.get("GRAMMAR_FIX_INPUT") or "true").strip().lower()
 GRAMMAR_FIX_INPUT = _gfi not in {"0", "false", "no", "off"}
 _gfo = (os.environ.get("GRAMMAR_FIX_OUTPUT") or "true").strip().lower()
 GRAMMAR_FIX_OUTPUT = _gfo not in {"0", "false", "no", "off"}
+
+# Classical lexicon path: only used when generative is unavailable / ML polish off
+_lex = (os.environ.get("LEXICON_FALLBACK") or "true").strip().lower()
+LEXICON_FALLBACK = _lex not in {"0", "false", "no", "off"}

app/data/content_swaps.json

@@ -19,8 +19,6 @@
   "shoots": "sprouts",
   "rays": "beams",
   "ray": "beam",
-  "time": "moment",
-  "times": "moments",
   "happy": "glad",
   "quickly": "swiftly",
   "slowly": "gradually",
@@ -165,7 +163,6 @@
   "different": "distinct",
   "whole": "entire",
   "half": "partial",
-  "enough": "adequate",
   "several": "various",
   "certain": "particular",
   "possible": "feasible",
@@ -193,7 +190,6 @@
   "first": "initial",
   "second": "next",
   "third": "another",
-
   "change": "shift",
   "changes": "shifts",
   "changed": "shifted",
@@ -228,7 +224,6 @@
   "protect": "safeguard",
   "protects": "safeguards",
   "protected": "safeguarded",
-  "people": "communities",
   "environment": "ecosystem",
   "agriculture": "farming",
   "farmers": "growers",
@@ -324,15 +319,12 @@
   "factors": "elements",
   "example": "instance",
   "examples": "instances",
-  "today": "nowadays",
-  "nowadays": "today",
   "often": "frequently",
   "rarely": "seldom",
   "always": "consistently",
   "clearly": "plainly",
   "simply": "merely",
   "really": "truly",
-  "very": "highly",
   "quite": "fairly",
   "rather": "somewhat",
   "especially": "particularly",
@@ -374,7 +366,6 @@
   "connected": "linked",
   "similar": "alike",
   "various": "assorted",
-  "common": "usual",
   "unique": "distinct",
   "complex": "intricate",
   "simple": "basic",
@@ -391,11 +382,7 @@
   "serious": "severe",
   "severe": "serious",
   "strong": "sturdy",
-
   "reducing": "lowering",
-  "reduce": "lower",
-  "reduces": "lowers",
-  "reduced": "lowered",
   "improving": "enhancing",
   "improve": "enhance",
   "improves": "enhances",
@@ -403,5 +390,6 @@
   "costs": "expenses",
   "cost": "expense",
   "bugs": "defects",
-  "bug": "defect"
+  "bug": "defect",
+  "common": "widespread"
 }

app/data/elevate_swaps.json

@@ -69,7 +69,6 @@
   "about": "regarding",
   "so": "therefore",
   "also": "additionally",
-  "people": "individuals",
   "person": "individual",
   "change": "modification",
   "changes": "modifications",

app/data/protected_phrases.json

@@ -9,6 +9,10 @@
   "as many",
   "how far",
   "how long",
+  "enough time",
+  "enough times",
+  "very common",
+  "more common",
   "fast food",
   "fast foods",
   "junk food",

app/main.py

@@ -128,7 +128,14 @@ def health():
                 "minilm": "fastembed|sentence-transformers" if ml_pkg else None,
                 "generative": GENERATIVE_MODEL if gen_pkg else None,
             },
-            "pipeline": ["flan-t5 paraphrase", "rules", "minilm meaning guard"],
+            "pipeline": [
+                "grammar",
+                "generative paraphrase",
+                "validate/rank",
+                "rules-light",
+                "minilm meaning guard",
+                "lexicon-fallback",
+            ],
             "plans": "all",
         },
     }

app/pipeline/candidate_ranker.py

@@ -0,0 +1,99 @@
+"""Rank validated rewrite candidates for fidelity + natural difference."""
+
+from __future__ import annotations
+
+from difflib import SequenceMatcher
+
+from app.pipeline.candidate_validator import ValidationResult, filter_valid_candidates
+from app.pipeline.tones import normalize_tone
+
+
+def _surface_sim(a: str, b: str) -> float:
+    return SequenceMatcher(None, a.lower().strip(), b.lower().strip()).ratio()
+
+
+def _tone_bonus(candidate: str, tone: str) -> float:
+    """Light register heuristics — secondary to meaning."""
+    tone_l = normalize_tone(tone)
+    low = candidate.lower()
+    casual_marks = ("don't", "can't", "won't", "it's", "that's", "gonna", "kinda")
+    formal_marks = ("therefore", "however", "moreover", "furthermore", "consequently")
+    academic_marks = ("thus", "hence", "whereby", "notwithstanding", "aforementioned")
+    if tone_l == "Casual":
+        return 0.08 if any(m in low for m in casual_marks) else 0.0
+    if tone_l == "Formal":
+        return 0.08 if any(m in low for m in formal_marks) else 0.0
+    if tone_l == "Academic":
+        return 0.08 if any(m in low for m in academic_marks + formal_marks) else 0.0
+    return 0.0
+
+
+def score_candidate(
+    original: str,
+    candidate: str,
+    validation: ValidationResult,
+    *,
+    tone: str = "Neutral",
+) -> float:
+    """
+    Higher is better.
+
+    Weights favor meaning preservation over "being different".
+    """
+    meaning = validation.meaning
+    surf = validation.surface_sim if validation.surface_sim else _surface_sim(original, candidate)
+    # Useful difference: reward leaving the near-copy zone, not extreme divergence
+    if surf >= 0.95:
+        diff = 0.0
+    elif surf >= 0.80:
+        diff = 0.35
+    elif surf >= 0.55:
+        diff = 1.0
+    elif surf >= 0.35:
+        diff = 0.7
+    else:
+        diff = 0.25
+
+    ow = max(1, len(original.split()))
+    cw = len(candidate.split())
+    length_ratio = cw / ow
+    if 0.75 <= length_ratio <= 1.35:
+        length_score = 1.0
+    elif 0.55 <= length_ratio <= 1.7:
+        length_score = 0.6
+    else:
+        length_score = 0.2
+
+    tone_b = _tone_bonus(candidate, tone)
+
+    return (
+        0.45 * meaning
+        + 0.25 * diff
+        + 0.15 * length_score
+        + 0.10 * (1.0 - abs(surf - 0.65))  # soft preference for moderate rewrite
+        + 0.05
+        + tone_b
+    )
+
+
+def pick_best_candidate(
+    original: str,
+    candidates: list[str],
+    *,
+    tone: str = "Neutral",
+    min_meaning: float = 0.72,
+    fallback: str | None = None,
+) -> str:
+    """Validate + rank; return best candidate or fallback (default: original)."""
+    valid = filter_valid_candidates(original, candidates, min_meaning=min_meaning)
+    if not valid:
+        return fallback if fallback is not None else original
+
+    best_text = valid[0][0]
+    best_score = -1.0
+    for text, result in valid:
+        s = score_candidate(original, text, result, tone=tone)
+        if s > best_score:
+            best_score = s
+            best_text = text
+    return best_text

app/pipeline/candidate_validator.py

@@ -0,0 +1,165 @@
+"""Validate rewrite candidates for meaning, polarity, and fact preservation."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from app.pipeline.meaning_safety import polarity_safe
+
+# Tokens that look like proper nouns / identifiers to preserve.
+_PROPER = re.compile(r"\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)*)\b")
+_NUMBER = re.compile(
+    r"\b(?:\d{1,3}(?:,\d{3})+(?:\.\d+)?|\d+\.\d+%?|\d+%?|\d{4})\b"
+)
+_QUOTED = re.compile(r"[\"“”']([^\"“”']{2,120})[\"“”']")
+_YEAR = re.compile(r"\b(?:19|20)\d{2}\b")
+
+
+@dataclass
+class ValidationResult:
+    ok: bool
+    reasons: list[str]
+    meaning: float = 1.0
+    surface_sim: float = 1.0
+
+
+def _surface_sim(a: str, b: str) -> float:
+    from difflib import SequenceMatcher
+
+    return SequenceMatcher(None, a.lower().strip(), b.lower().strip()).ratio()
+
+
+def _extract_numbers(text: str) -> set[str]:
+    return {m.group(0) for m in _NUMBER.finditer(text)} | {
+        m.group(0) for m in _YEAR.finditer(text)
+    }
+
+
+def _extract_quotes(text: str) -> set[str]:
+    return {m.group(1).strip().lower() for m in _QUOTED.finditer(text)}
+
+
+def _extract_propers(text: str) -> set[str]:
+    """Capitalized multi-word / mid-sentence names (skip sentence starts)."""
+    found: set[str] = set()
+    for m in _PROPER.finditer(text):
+        start = m.start()
+        # Skip if at start or after sentence boundary (likely sentence capitalization)
+        if start == 0:
+            continue
+        prev = text[start - 1]
+        if prev in ".!?\n":
+            continue
+        found.add(m.group(1))
+    return found
+
+
+def _length_ok(original: str, candidate: str) -> bool:
+    ow = max(1, len(original.split()))
+    cw = len(candidate.split())
+    if cw < max(3, ow // 3):
+        return False
+    if cw > int(ow * 2.2) + 8:
+        return False
+    return True
+
+
+def _meaning_score(original: str, candidate: str) -> float | None:
+    """MiniLM cosine if available; None if model missing."""
+    try:
+        from app.pipeline.minilm import _cosine, _embed_texts, _ensure_model
+
+        if _ensure_model() is None:
+            return None
+        vecs = _embed_texts([original, candidate])
+        if len(vecs) < 2:
+            return None
+        return float(_cosine(vecs[0], vecs[1]))
+    except Exception:
+        return None
+
+
+def validate_candidate(
+    original: str,
+    candidate: str,
+    *,
+    min_meaning: float = 0.72,
+    max_surface: float = 0.97,
+    min_surface: float = 0.25,
+) -> ValidationResult:
+    """Return whether a candidate is safe to keep as a rewrite of `original`."""
+    reasons: list[str] = []
+    o = (original or "").strip()
+    c = (candidate or "").strip()
+    if not o or not c:
+        return ValidationResult(False, ["empty"], 0.0, 0.0)
+
+    if c == o:
+        return ValidationResult(False, ["identical"], 1.0, 1.0)
+
+    if not polarity_safe(o, c):
+        reasons.append("polarity")
+
+    if not _length_ok(o, c):
+        reasons.append("length")
+
+    surf = _surface_sim(o, c)
+    if surf >= max_surface:
+        reasons.append("too_similar")
+    if surf < min_surface and len(o.split()) >= 8:
+        reasons.append("too_divergent")
+
+    o_nums, c_nums = _extract_numbers(o), _extract_numbers(c)
+    if o_nums and not o_nums.issubset(c_nums):
+        # Allow reformatting like 1,000 vs 1000 by normalizing commas
+        o_norm = {n.replace(",", "") for n in o_nums}
+        c_norm = {n.replace(",", "") for n in c_nums}
+        if not o_norm.issubset(c_norm):
+            reasons.append("numbers")
+
+    o_quotes = _extract_quotes(o)
+    c_quotes = _extract_quotes(c)
+    if o_quotes and not o_quotes.issubset(c_quotes):
+        reasons.append("quotes")
+
+    for name in _extract_propers(o):
+        if name not in c and name.lower() not in c.lower():
+            reasons.append(f"entity:{name}")
+            break
+
+    meaning = _meaning_score(o, c)
+    if meaning is None:
+        meaning = 1.0 if not reasons else 0.5
+    elif meaning < min_meaning:
+        reasons.append(f"meaning:{meaning:.2f}")
+
+    return ValidationResult(
+        ok=not reasons,
+        reasons=reasons,
+        meaning=float(meaning),
+        surface_sim=surf,
+    )
+
+
+def filter_valid_candidates(
+    original: str,
+    candidates: list[str],
+    *,
+    min_meaning: float = 0.72,
+) -> list[tuple[str, ValidationResult]]:
+    """Deduplicate and keep only candidates that pass validation."""
+    seen: set[str] = set()
+    out: list[tuple[str, ValidationResult]] = []
+    for raw in candidates:
+        text = re.sub(r"\s+", " ", (raw or "").strip())
+        if not text:
+            continue
+        key = text.lower()
+        if key in seen:
+            continue
+        seen.add(key)
+        result = validate_candidate(original, text, min_meaning=min_meaning)
+        if result.ok:
+            out.append((text, result))
+    return out

app/pipeline/generative.py

@@ -1,11 +1,11 @@
-"""Aggressive FLAN-T5 paraphrase for ML polish (main rewrite path).
+"""Paragraph-level generative rewrite (quality-first path).
 
-When ML polish is on and the model loads:
-  1) This module drafts a clearly reworded, tone-aware paraphrase
-  2) Classical rules lightly polish (scrub / register)
-  3) MiniLM only blocks severe meaning drift
+Pipeline when models load:
+  1) Draft several paraphrases per paragraph (seq2seq)
+  2) Validate polarity / facts / meaning
+  3) Rank survivors — never pick "most different" alone
 
-Lazy-loaded; missing torch/transformers → callers skip to rules-only.
+Lazy-loaded; missing torch/transformers → callers fall back to classical rules.
 """
 
 from __future__ import annotations
@@ -13,17 +13,16 @@ from __future__ import annotations
 import logging
 import re
 import threading
-from difflib import SequenceMatcher
 from typing import Any
 
 from app.config import (
-    GENERATIVE_MAX_SENTENCES,
+    GENERATIVE_MAX_PARAGRAPHS,
     GENERATIVE_MODEL,
     GENERATIVE_POLISH_ENABLED,
     ML_POLISH_AVAILABLE_DEFAULT,
 )
-from app.pipeline.normalize import split_paragraphs, split_sentences_regex
-from app.pipeline.nlp import get_nlp
+from app.pipeline.candidate_ranker import pick_best_candidate
+from app.pipeline.normalize import split_paragraphs
 from app.pipeline.tones import normalize_tone
 
 logger = logging.getLogger("plainrewrite.generative")
@@ -34,43 +33,48 @@ _model: Any = None
 _failed = False
 _backend: str | None = None
 
-# Demand visible rewording — FLAN-T5 often copies unless pushed hard.
 _TONE_PROMPTS: dict[str, str] = {
     "Casual": (
-        "Paraphrase the sentence using different words and a casual spoken style. "
-        "Change sentence structure if helpful. Keep the same facts. "
-        "Do not copy phrases verbatim.\n"
-        "Input: {text}\n"
-        "Paraphrase:"
+        "Rewrite the paragraph in clear casual English.\n"
+        "Keep every claim and the same polarity. Do not flip negatives or antonyms.\n"
+        "Preserve names, numbers, dates, and quotations. Correct grammar.\n"
+        "Improve flow; do not invent facts. Return only the rewritten paragraph.\n\n"
+        "Paragraph:\n{text}\n\n"
+        "Rewrite:"
     ),
     "Formal": (
-        "Paraphrase the sentence using different words and a formal professional style. "
-        "Change sentence structure if helpful. Keep the same facts. "
-        "Do not copy phrases verbatim.\n"
-        "Input: {text}\n"
-        "Paraphrase:"
+        "Rewrite the paragraph in clear formal professional English.\n"
+        "Keep every claim and the same polarity. Do not flip negatives or antonyms.\n"
+        "Preserve names, numbers, dates, and quotations. Correct grammar.\n"
+        "Improve flow; do not invent facts. Return only the rewritten paragraph.\n\n"
+        "Paragraph:\n{text}\n\n"
+        "Rewrite:"
     ),
     "Academic": (
-        "Paraphrase the sentence using different words and a clear academic style. "
-        "Change sentence structure if helpful. Keep the same facts. "
-        "Do not copy phrases verbatim.\n"
-        "Input: {text}\n"
-        "Paraphrase:"
+        "Rewrite the paragraph in clear academic English.\n"
+        "Keep every claim and the same polarity. Do not flip negatives or antonyms.\n"
+        "Preserve names, numbers, dates, and quotations. Correct grammar.\n"
+        "Improve flow; do not invent facts. Return only the rewritten paragraph.\n\n"
+        "Paragraph:\n{text}\n\n"
+        "Rewrite:"
     ),
     "Neutral": (
-        "Paraphrase the sentence using different words while keeping the same meaning. "
-        "Change wording and structure clearly. Do not copy phrases verbatim.\n"
-        "Input: {text}\n"
-        "Paraphrase:"
+        "Rewrite the paragraph in clear natural English.\n"
+        "Keep every claim and the same polarity (e.g. unhealthy stays unhealthy; "
+        "don't realize stays a lack of awareness).\n"
+        "Preserve names, numbers, dates, and quotations. Correct grammar.\n"
+        "Improve sentence structure and flow. Do not invent facts.\n"
+        "Return only the rewritten paragraph.\n\n"
+        "Paragraph:\n{text}\n\n"
+        "Rewrite:"
     ),
 }
 
 _RETRY_PROMPT = (
-    "Write a clearly different paraphrase of this sentence. "
-    "Replace most content words with synonyms and reorder clauses. "
-    "Keep the meaning. Tone: {tone}.\n"
-    "Input: {text}\n"
-    "New paraphrase:"
+    "Rewrite this paragraph with different wording but the exact same meaning "
+    "and polarity. Do not add or remove facts. Tone: {tone}.\n\n"
+    "Paragraph:\n{text}\n\n"
+    "Rewrite:"
 )
 
 
@@ -117,17 +121,17 @@ def _ensure_model():
             from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
 
             name = GENERATIVE_MODEL
-            logger.info("Loading generative polish model %s …", name)
+            logger.info("Loading generative rewrite model %s …", name)
             _tokenizer = AutoTokenizer.from_pretrained(name)
             _model = AutoModelForSeq2SeqLM.from_pretrained(name)
             _model.eval()
             _model.to("cpu")
             _backend = f"transformers:{name}"
             _ = torch.__version__
-            logger.info("Generative polish ready (%s)", _backend)
+            logger.info("Generative rewrite ready (%s)", _backend)
             return _model
         except Exception as exc:
-            logger.warning("Generative polish unavailable: %s", exc)
+            logger.warning("Generative rewrite unavailable: %s", exc)
             _failed = True
             _tokenizer = None
             _model = None
@@ -135,112 +139,47 @@ def _ensure_model():
             return None
 
 
-def _split_sents(text: str) -> list[str]:
-    nlp = get_nlp()
-    if nlp is not None:
-        return [s.text.strip() for s in nlp(text).sents if s.text.strip()]
-    return [s for s in split_sentences_regex(text) if s.strip()]
-
-
-def _sim(a: str, b: str) -> float:
-    return SequenceMatcher(None, a.lower().strip(), b.lower().strip()).ratio()
-
-
 def _clean_gen_text(text: str) -> str:
     text = re.sub(r"\s+", " ", (text or "").strip())
     if not text:
         return ""
     low = text.lower()
-    for prefix in ("paraphrase:", "rewrite:", "new paraphrase:", "input:", "sentence:", "output:"):
+    for prefix in (
+        "rewrite:",
+        "paraphrase:",
+        "new paraphrase:",
+        "input:",
+        "paragraph:",
+        "sentence:",
+        "output:",
+    ):
         if low.startswith(prefix):
             text = text[len(prefix) :].strip()
             low = text.lower()
     return text
 
 
-def _gen_kwargs(strength: int, *, aggressive: bool = False) -> dict[str, Any]:
-    """Sampling-heavy decode so paraphrases diverge from the source."""
+def _gen_kwargs(strength: int, *, n_return: int | None = None) -> dict[str, Any]:
     strength = max(0, min(2, int(strength)))
-    if aggressive:
-        return {
-            "do_sample": True,
-            "num_beams": 1,
-            "temperature": 1.15,
-            "top_p": 0.92,
-            "top_k": 50,
-            "max_new_tokens": 160,
-            "no_repeat_ngram_size": 3,
-            "num_return_sequences": 3,
-        }
+    base = {
+        0: {"temperature": 0.75, "top_p": 0.9, "top_k": 40, "max_new_tokens": 192, "n": 3},
+        1: {"temperature": 0.9, "top_p": 0.92, "top_k": 50, "max_new_tokens": 256, "n": 4},
+        2: {"temperature": 1.0, "top_p": 0.94, "top_k": 55, "max_new_tokens": 320, "n": 5},
+    }[strength]
+    n = n_return if n_return is not None else base["n"]
     return {
-        0: {
-            "do_sample": True,
-            "num_beams": 1,
-            "temperature": 0.85,
-            "top_p": 0.9,
-            "top_k": 40,
-            "max_new_tokens": 96,
-            "no_repeat_ngram_size": 3,
-            "num_return_sequences": 2,
-        },
-        1: {
-            "do_sample": True,
-            "num_beams": 1,
-            "temperature": 1.0,
-            "top_p": 0.92,
-            "top_k": 50,
-            "max_new_tokens": 128,
-            "no_repeat_ngram_size": 3,
-            "num_return_sequences": 3,
-        },
-        2: {
-            "do_sample": True,
-            "num_beams": 1,
-            "temperature": 1.15,
-            "top_p": 0.95,
-            "top_k": 60,
-            "max_new_tokens": 160,
-            "no_repeat_ngram_size": 3,
-            "num_return_sequences": 4,
-        },
-    }.get(
-        strength,
-        {
-            "do_sample": True,
-            "num_beams": 1,
-            "temperature": 1.0,
-            "top_p": 0.92,
-            "max_new_tokens": 128,
-            "num_return_sequences": 3,
-        },
-    )
-
-
-def _pick_most_different(original: str, candidates: list[str], *, max_sim: float) -> str:
-    """Prefer the candidate farthest from the original that still looks like a sentence."""
-    best: str | None = None
-    best_score = 2.0
-    for raw in candidates:
-        c = _clean_gen_text(raw)
-        if len(c) < 3:
-            continue
-        if len(c.split()) < max(3, len(original.split()) // 4):
-            continue
-        score = _sim(original, c)
-        if score < best_score:
-            best_score = score
-            best = c
-    if best is None:
-        return original
-    # Accept best even if above max_sim when it's clearly not identical
-    if best_score >= 0.97:
-        return original
-    if best_score > max_sim and best_score > 0.92:
-        return original
-    return best
-
-
-def _generate_once(prompt: str, strength: int, *, aggressive: bool) -> list[str]:
+        "do_sample": True,
+        "num_beams": 1,
+        "temperature": base["temperature"],
+        "top_p": base["top_p"],
+        "top_k": base["top_k"],
+        "max_new_tokens": base["max_new_tokens"],
+        "no_repeat_ngram_size": 3,
+        "num_return_sequences": max(2, min(int(n), 5)),
+    }
+
+
+def _generate_once(prompt: str, strength: int, *, n_return: int | None = None) -> list[str]:
     model = _ensure_model()
     if model is None or _tokenizer is None:
         return []
@@ -251,47 +190,79 @@ def _generate_once(prompt: str, strength: int, *, aggressive: bool) -> list[str]
             prompt,
             return_tensors="pt",
             truncation=True,
-            max_length=384,
+            max_length=512,
         )
-        kwargs = _gen_kwargs(strength, aggressive=aggressive)
+        kwargs = _gen_kwargs(strength, n_return=n_return)
         with torch.no_grad():
             out_ids = model.generate(**inputs, **kwargs)
-        texts: list[str] = []
-        for row in out_ids:
-            texts.append(_tokenizer.decode(row, skip_special_tokens=True))
-        return texts
+        return [_tokenizer.decode(row, skip_special_tokens=True) for row in out_ids]
     except Exception as exc:
         logger.warning("Generative decode failed: %s", exc)
         return []
 
 
-def _paraphrase_sentence(
-    sentence: str,
+def _chunk_long_paragraph(para: str, max_words: int = 180) -> list[str]:
+    """Split oversized paragraphs into sentence groups for model context limits."""
+    words = para.split()
+    if len(words) <= max_words:
+        return [para]
+    # Prefer sentence boundaries
+    parts = re.split(r"(?<=[.!?])\s+", para.strip())
+    chunks: list[str] = []
+    buf: list[str] = []
+    count = 0
+    for sent in parts:
+        w = len(sent.split())
+        if buf and count + w > max_words:
+            chunks.append(" ".join(buf))
+            buf = [sent]
+            count = w
+        else:
+            buf.append(sent)
+            count += w
+    if buf:
+        chunks.append(" ".join(buf))
+    return chunks or [para]
+
+
+def _paraphrase_paragraph(
+    paragraph: str,
     tone: str,
     strength: int,
     *,
-    max_sim: float = 0.88,
+    min_meaning: float = 0.72,
 ) -> str:
-    """Return a paraphrase that differs enough from the source, or the source."""
+    """Generate candidates for one paragraph; return best validated rewrite or original."""
     model = _ensure_model()
     if model is None or _tokenizer is None:
-        return sentence
+        return paragraph
+    if len(paragraph.split()) < 3:
+        return paragraph
 
     tone_l = normalize_tone(tone)
-    prompt = _TONE_PROMPTS.get(tone_l, _TONE_PROMPTS["Neutral"]).format(text=sentence.strip())
-    cands = _generate_once(prompt, strength, aggressive=False)
-    picked = _pick_most_different(sentence, cands, max_sim=max_sim)
-
-    # Retry harder if still too similar
-    if picked == sentence or _sim(sentence, picked) > max_sim:
-        retry = _RETRY_PROMPT.format(tone=tone_l.lower(), text=sentence.strip())
-        cands2 = _generate_once(retry, min(2, strength + 1), aggressive=True)
-        picked2 = _pick_most_different(sentence, cands2, max_sim=max_sim)
-        if picked2 != sentence and _sim(sentence, picked2) < _sim(sentence, picked):
-            picked = picked2
-
-    if picked == sentence or _sim(sentence, picked) > 0.97:
-        return sentence
+    prompt = _TONE_PROMPTS.get(tone_l, _TONE_PROMPTS["Neutral"]).format(text=paragraph.strip())
+    raw = _generate_once(prompt, strength)
+    cands = [_clean_gen_text(x) for x in raw]
+
+    # Second pass if nothing useful
+    picked = pick_best_candidate(
+        paragraph,
+        cands,
+        tone=tone_l,
+        min_meaning=min_meaning,
+        fallback=paragraph,
+    )
+    if picked == paragraph:
+        retry = _RETRY_PROMPT.format(tone=tone_l.lower(), text=paragraph.strip())
+        raw2 = _generate_once(retry, min(2, strength + 1), n_return=5)
+        cands2 = [_clean_gen_text(x) for x in raw2]
+        picked = pick_best_candidate(
+            paragraph,
+            cands + cands2,
+            tone=tone_l,
+            min_meaning=min_meaning,
+            fallback=paragraph,
+        )
     return picked
 
 
@@ -301,11 +272,14 @@ def generative_paraphrase(
     tone: str = "Neutral",
     strength: int = 1,
     max_sim: float = 0.88,
+    min_meaning: float = 0.72,
 ) -> str | None:
     """
-    Aggressive sentence-wise paraphrase with FLAN-T5.
+    Quality-first paragraph paraphrase.
     Returns None if the model is unavailable.
+    `max_sim` kept for API compatibility (ranking handles similarity).
     """
+    del max_sim  # ranking/validation supersede surface-only max_sim
     if not text.strip():
         return None
     if _ensure_model() is None:
@@ -313,50 +287,42 @@ def generative_paraphrase(
 
     tone = normalize_tone(tone)
     strength = max(0, min(2, int(strength)))
-    # ML polish defaults to at least Normal generative force
     gen_strength = max(1, strength)
 
     paras = split_paragraphs(text)
     out_paras: list[str] = []
-    # 0 / unset = entire uploaded document; positive = optional safety cap
-    budget = int(GENERATIVE_MAX_SENTENCES)
+    budget = int(GENERATIVE_MAX_PARAGRAPHS)
     unlimited = budget <= 0
     used = 0
     changed_n = 0
 
     for para in paras:
-        sents = _split_sents(para)
-        if not sents:
+        if not para.strip():
             continue
-        new_sents: list[str] = []
-        for sent in sents:
+        chunks = _chunk_long_paragraph(para)
+        new_chunks: list[str] = []
+        for chunk in chunks:
             if not unlimited and used >= budget:
-                new_sents.append(sent)
-                continue
-            if len(sent.split()) < 3:
-                new_sents.append(sent)
+                new_chunks.append(chunk)
                 continue
-            nxt = _paraphrase_sentence(sent, tone, gen_strength, max_sim=max_sim)
+            nxt = _paraphrase_paragraph(
+                chunk, tone, gen_strength, min_meaning=min_meaning
+            )
             used += 1
-            if nxt.strip() != sent.strip() and _sim(sent, nxt) <= max_sim:
-                changed_n += 1
-                new_sents.append(nxt)
-            elif nxt.strip() != sent.strip():
-                # Accept mild change over none
+            if nxt.strip() != chunk.strip():
                 changed_n += 1
-                new_sents.append(nxt)
-            else:
-                new_sents.append(sent)
-        out_paras.append(" ".join(new_sents))
+            new_chunks.append(nxt)
+        out_paras.append(" ".join(new_chunks))
 
     if not out_paras:
         return None
     result = "\n\n".join(p for p in out_paras if p)
     logger.info(
-        "Generative paraphrase: %s/%s sentences changed (strength=%s, budget=%s)",
+        "Generative paraphrase: %s/%s units changed (strength=%s, model=%s, budget=%s)",
         changed_n,
         used,
         gen_strength,
+        GENERATIVE_MODEL,
         "all" if unlimited else budget,
     )
     return result
@@ -366,7 +332,11 @@ def warm_generative() -> bool:
     try:
         ok = generative_available()
         if ok:
-            _paraphrase_sentence("The system works well for most users.", "Neutral", 1)
+            _paraphrase_paragraph(
+                "The system works well for most users and saves time.",
+                "Neutral",
+                1,
+            )
         return ok
     except Exception as exc:
         logger.warning("Generative warm failed: %s", exc)

app/pipeline/grammar_fix.py

@@ -95,6 +95,28 @@ def _fix_mass_nouns(text: str) -> str:
     return pattern.sub(repl, text)
 
 
+def _fix_count_noun_slips(text: str) -> str:
+    """Fix common ESL count/mass slips that aren't full plurals."""
+    # "enough times" (duration) → "enough time"
+    text = re.sub(r"\benough\s+times\b", "enough time", text, flags=re.I)
+    # "Eating X are" / "X foods are becoming" → singular verb with plural subject of gerund
+    text = re.sub(
+        r"\b(Eating\s+(?:fast\s+)?foods)\s+are\b",
+        r"\1 is",
+        text,
+        flags=re.I,
+    )
+    # "living unhealthy life" → "living an unhealthy life"
+    text = re.sub(
+        r"\b(living|live|lived|leads?|led)\s+(unhealthy|healthy|better|good|bad)\s+life\b",
+        lambda m: f"{m.group(1)} {'an' if m.group(2).lower()[0] in 'aeiou' else 'a'} "
+        f"{m.group(2)} life",
+        text,
+        flags=re.I,
+    )
+    return text
+
+
 def _fix_do_support(text: str) -> str:
     """`don't realizes` → `don't realize`; `does works` → `does work`."""
     aux = r"(do|does|did|don't|doesn't|didn't|do not|does not|did not)"
@@ -285,11 +307,12 @@ def correct_text(text: str, language: str | None = None) -> str:
     try:
         out = _apply_languagetool(text, language)
         out = _fix_mass_nouns(out)
+        out = _fix_count_noun_slips(out)
         out = _fix_do_support(out)
         if get_nlp() is not None:
             out = _fix_agreement_spacy(out)
-        else:
-            out = _fix_agreement_regex(out)
+        # Regex pronoun patterns catch cases spaCy misses (e.g. "how much it affect").
+        out = _fix_agreement_regex(out)
         return out
     except Exception as exc:  # noqa: BLE001
         logger.warning("correct_text failed, returning original: %s", exc)

app/pipeline/meaning_safety.py

@@ -0,0 +1,101 @@
+"""Lightweight meaning-safety checks for rewrite candidates.
+
+Used by FLAN-T5 candidate picking and MiniLM keep/revert so antonym flips
+and negation reversals (unhealthy→healthy, don't realize→realize) are rejected.
+"""
+
+from __future__ import annotations
+
+import re
+
+# Prefixed adjective flips that reverse sentiment / claim polarity.
+_PREFIX_FLIPS: tuple[tuple[str, str], ...] = (
+    ("unhealthy", "healthy"),
+    ("unhappy", "happy"),
+    ("unsafe", "safe"),
+    ("unclear", "clear"),
+    ("unusual", "usual"),
+    ("unimportant", "important"),
+    ("unlikely", "likely"),
+    ("unable", "able"),
+    ("incorrect", "correct"),
+    ("incomplete", "complete"),
+    ("indirect", "direct"),
+    ("illegal", "legal"),
+    ("irresponsible", "responsible"),
+    ("dishonest", "honest"),
+    ("disloyal", "loyal"),
+    ("disagree", "agree"),
+    ("decrease", "increase"),
+    ("worsen", "improve"),
+    ("worse", "better"),
+    ("bad", "good"),
+    ("negative", "positive"),
+    ("harmful", "helpful"),
+    ("dangerous", "safe"),
+)
+
+# Content verbs whose negation must not flip.
+_NEGATABLE_VERBS = (
+    "realize", "realise", "know", "understand", "notice", "see", "feel",
+    "want", "need", "like", "agree", "believe", "think", "have", "help",
+    "cause", "affect", "support", "allow", "permit", "prevent",
+)
+
+_NEG_BEFORE = re.compile(
+    r"\b(?:do|does|did|can|could|will|would|should|must|shall|may|might)\s+not\b|"
+    r"\b(?:don't|doesn't|didn't|can't|couldn't|won't|wouldn't|shouldn't|mustn't|"
+    r"isn't|aren't|wasn't|weren't|hasn't|haven't|hadn't|never|no)\b",
+    re.I,
+)
+
+
+def _contains_word(text: str, word: str) -> bool:
+    return bool(re.search(rf"\b{re.escape(word)}\b", text, flags=re.I))
+
+
+def _negation_before_verb(text: str, verb: str) -> bool:
+    """True if a negation marker appears shortly before the verb lemma."""
+    # Match don't/does not/never + optional words + verb forms
+    pat = re.compile(
+        rf"(?:don't|doesn't|didn't|do\s+not|does\s+not|did\s+not|never|not)\s+"
+        rf"(?:\w+\s+){{0,2}}{re.escape(verb)}\w*",
+        re.I,
+    )
+    return bool(pat.search(text))
+
+
+def polarity_safe(original: str, candidate: str) -> bool:
+    """Reject candidates that flip antonyms or strip/add negation on key verbs."""
+    if not original.strip() or not candidate.strip():
+        return False
+    o, c = original.lower(), candidate.lower()
+    if o == c:
+        return True
+
+    for a, b in _PREFIX_FLIPS:
+        # orig has A, cand has B without A → flipped
+        if _contains_word(o, a) and _contains_word(c, b) and not _contains_word(c, a):
+            return False
+        if _contains_word(o, b) and _contains_word(c, a) and not _contains_word(c, b):
+            return False
+
+    for verb in _NEGATABLE_VERBS:
+        o_neg = _negation_before_verb(o, verb)
+        c_neg = _negation_before_verb(c, verb)
+        o_has = bool(re.search(rf"\b{re.escape(verb)}\w*\b", o))
+        c_has = bool(re.search(rf"\b{re.escape(verb)}\w*\b", c))
+        if o_has and c_has and o_neg != c_neg:
+            return False
+
+    # Overall negation density shouldn't invert sharply on short sentences
+    o_negs = len(_NEG_BEFORE.findall(o))
+    c_negs = len(_NEG_BEFORE.findall(c))
+    if o_negs >= 1 and c_negs == 0 and len(o.split()) <= 40:
+        # Candidate dropped all negations — likely a flip
+        # Allow if original's only negation is in a protected idiom we can't check;
+        # still reject when a known flip verb was negated.
+        if any(_negation_before_verb(o, v) for v in _NEGATABLE_VERBS):
+            return False
+
+    return True

app/pipeline/ml_polish.py

@@ -2,13 +2,15 @@
 
 from __future__ import annotations
 
+from app.pipeline.meaning_safety import polarity_safe
 from app.pipeline.ml_context import mark_ml_used, ml_polish_enabled
 from app.pipeline.minilm import _cosine, _embed_texts, _ensure_model
 from app.pipeline.normalize import split_paragraphs, split_sentences_regex
 from app.pipeline.nlp import get_nlp
 
-# Default floor for rules-only path. Generative path uses a looser floor.
-_MIN_MEANING_KEEP = 0.68
+# Rules path. Generative path uses a slightly lower floor but still meaning-strict.
+_MIN_MEANING_KEEP = 0.72
+_MIN_MEANING_GENERATIVE = 0.70
 
 
 def _split_sents(text: str) -> list[str]:
@@ -29,8 +31,11 @@ def _meaning_ok(
         return False
     if original.strip() == candidate.strip():
         return True
+    if not polarity_safe(original, candidate):
+        return False
     if _ensure_model() is None:
-        return True  # no model → trust the rewrite
+        # No MiniLM — still enforce polarity; accept otherwise
+        return True
     vecs = _embed_texts([original, candidate])
     if len(vecs) < 2:
         return True

app/pipeline/orchestrator.py

@@ -1,4 +1,8 @@
-"""Orchestrate non-LLM rewrite pipeline (+ optional ML polish)."""
+"""Orchestrate quality-first rewrite: grammar → generative → validate → grammar.
+
+Classical lexicon/WordNet is demoted to offline fallback when generative is
+unavailable or ML polish is off.
+"""
 
 from __future__ import annotations
 
@@ -8,7 +12,14 @@ import time
 from dataclasses import dataclass
 from difflib import SequenceMatcher
 
-from app.config import GRAMMAR_FIX_INPUT, GRAMMAR_FIX_OUTPUT
+from app.config import (
+    GENERATIVE_MODEL,
+    GENERATIVE_PRIMARY,
+    GRAMMAR_FIX_INPUT,
+    GRAMMAR_FIX_OUTPUT,
+    LEXICON_FALLBACK,
+)
+from app.pipeline.candidate_validator import validate_candidate
 from app.pipeline.generative import generative_available, generative_paraphrase
 from app.pipeline.grammar_fix import correct_text
 from app.pipeline.mechanics import enforce_length_budget, scrub_phrases, tidy
@@ -28,13 +39,13 @@ from app.pipeline.normalize import (
     word_count,
 )
 from app.pipeline.similarity import SimilarityResult, compare_similarity
-from app.pipeline.tones import normalize_tone
 from app.pipeline.synonym import rewrite_sentence_synonyms
 from app.pipeline.syntax_rewrite import (
     apply_tone_contractions,
     rewrite_paragraph_structure,
 )
 from app.pipeline.tone_style import apply_tone_style
+from app.pipeline.tones import normalize_tone
 
 logger = logging.getLogger("plainrewrite")
 if not logger.handlers:
@@ -55,7 +66,6 @@ class RewriteResult:
 
 
 def _rng_for(text: str) -> random.Random:
-    # Stable across processes (avoid PYTHONHASHSEED unpredictability)
     return random.Random(sum(ord(c) for c in text) * 2654435761 & 0xFFFFFFFF)
 
 
@@ -63,64 +73,103 @@ def _similarity_ratio(a: str, b: str) -> float:
     return SequenceMatcher(None, a.lower(), b.lower()).ratio()
 
 
-def _rewrite_paragraph(paragraph: str, tone: str, strength: int, rng: random.Random) -> str:
-    # Tone/phrases first so structure operates on elevated text for Academic/Heavy
+def _light_post_gen(text: str, tone: str) -> str:
+    """After generative rewrite: scrub fillers + contractions only — no synonym thrash."""
+    out = scrub_phrases(text)
+    out = apply_tone_contractions(out, tone)
+    return tidy(out)
+
+
+def _classical_paragraph(paragraph: str, tone: str, strength: int, rng: random.Random) -> str:
+    """Offline fallback: structure + curated lexicon (not the primary quality path)."""
     styled = apply_tone_style(paragraph, tone, strength, rng)
     structured = rewrite_paragraph_structure(styled, strength, rng, tone=tone)
-
     nlp = get_nlp()
     if nlp is not None:
         sents = [s.text.strip() for s in nlp(structured).sents if s.text.strip()]
     else:
         sents = split_sentences_regex(structured)
-
     rewritten_sents = [
         rewrite_sentence_synonyms(s, strength, rng, tone=tone) for s in sents
     ]
     joined = " ".join(rewritten_sents)
     joined = scrub_phrases(joined)
-    # Re-apply tone after scrub/synonym so register sticks
     joined = apply_tone_style(joined, tone, strength, rng)
     joined = apply_tone_contractions(joined, tone)
     return joined.strip()
 
 
-def _force_more_changes(text: str, tone: str, strength: int, rng: random.Random) -> str:
-    """Stronger per-paragraph pass — preserves \\n\\n breaks."""
-    bump = min(2, strength + 1)
-    paras = split_paragraphs(text)
-    out_paras: list[str] = []
-    for para in paras:
-        bumped = apply_tone_style(para, tone, bump, rng)
-        # Re-run structure transforms at higher strength for more sentence shape change
-        bumped = rewrite_paragraph_structure(bumped, bump, rng, tone=tone)
-        nlp = get_nlp()
-        if nlp is not None:
-            sents = [s.text.strip() for s in nlp(bumped).sents if s.text.strip()]
-        else:
-            sents = split_sentences_regex(bumped)
-        rewritten = [
-            rewrite_sentence_synonyms(s, bump, rng, tone=tone, force_all_lexicon=True)
-            for s in sents
-        ]
-        joined = scrub_phrases(" ".join(rewritten))
-        joined = apply_tone_style(joined, tone, bump, rng)
-        joined = apply_tone_contractions(joined, tone)
-        out_paras.append(joined.strip())
-    return "\n\n".join(p for p in out_paras if p)
-
-
-def _light_polish_after_gen(
-    text: str,
+def _classical_rewrite(
+    source: str,
     tone: str,
     strength: int,
     rng: random.Random,
+    *,
+    force: bool = False,
 ) -> str:
-    """After FLAN-T5, only scrub + register — avoid heavy synonym thrash that undoes the draft."""
-    out = scrub_phrases(text)
-    out = apply_tone_style(out, tone, strength, rng)
-    out = apply_tone_contractions(out, tone)
-    return tidy(out)
+    paragraphs = split_paragraphs(source)
+    out = [_classical_paragraph(p, tone, strength, rng) for p in paragraphs]
+    rewritten = tidy("\n\n".join(out))
+    rewritten = scrub_phrases(rewritten)
+    rewritten = apply_tone_style(rewritten, tone, strength, rng)
+    rewritten = apply_tone_contractions(rewritten, tone)
+    if force:
+        # Second-pass lexicon force for stubborn near-copies
+        bump = min(2, strength + 1)
+        paras = split_paragraphs(rewritten)
+        forced: list[str] = []
+        for para in paras:
+            bumped = apply_tone_style(para, tone, bump, rng)
+            bumped = rewrite_paragraph_structure(bumped, bump, rng, tone=tone)
+            nlp = get_nlp()
+            if nlp is not None:
+                sents = [s.text.strip() for s in nlp(bumped).sents if s.text.strip()]
+            else:
+                sents = split_sentences_regex(bumped)
+            rewritten_sents = [
+                rewrite_sentence_synonyms(
+                    s, bump, rng, tone=tone, force_all_lexicon=True
+                )
+                for s in sents
+            ]
+            joined = scrub_phrases(" ".join(rewritten_sents))
+            joined = apply_tone_contractions(joined, tone)
+            forced.append(joined.strip())
+        rewritten = tidy("\n\n".join(p for p in forced if p))
+    return rewritten
+
+
+def _accept_generative(source: str, draft: str) -> bool:
+    """Whole-document gate: polarity/meaning must hold vs grammar-corrected source."""
+    if not draft or not draft.strip():
+        return False
+
+    def _hard_fail(reasons: list[str]) -> bool:
+        for r in reasons:
+            if r == "polarity" or r in {"numbers", "quotes"}:
+                return True
+            if r.startswith("meaning") or r.startswith("entity"):
+                return True
+        return False
+
+    s_paras = split_paragraphs(source)
+    d_paras = split_paragraphs(draft)
+    if len(s_paras) == len(d_paras) and s_paras:
+        for sp, dp in zip(s_paras, d_paras):
+            if sp.strip() == dp.strip():
+                continue
+            v = validate_candidate(sp, dp, min_meaning=0.70)
+            if not v.ok and _hard_fail(v.reasons):
+                logger.info("Generative para rejected: %s", v.reasons)
+                return False
+        return True
+    v = validate_candidate(source, draft, min_meaning=0.70)
+    if v.ok or "identical" in v.reasons:
+        return True
+    if _hard_fail(v.reasons):
+        logger.info("Generative doc rejected: %s", v.reasons)
+        return False
+    return True
 
 
 def rewrite_text(
@@ -132,21 +181,18 @@ def rewrite_text(
     ml_polish: bool = False,
 ) -> RewriteResult:
     """
-    Classical rewrite by default.
+    Quality-first rewrite.
 
-    When ml_polish=True and models load:
-      1) Aggressive FLAN-T5 paraphrase (main rewrite)
-      2) Light rules polish (scrub / contractions / register)
-      3) MiniLM meaning guard vs original (loose floor — keep T5 unless severe drift)
-      If generative is unavailable → full classical rules + MiniLM ranking/guard.
+    When ml_polish=True and generative model loads:
+      grammar → paragraph generative (multi-candidate validate/rank) → light scrub → grammar
+    Otherwise (or if gen fails validation):
+      grammar → classical lexicon fallback (if LEXICON_FALLBACK) → grammar
     """
     started = time.perf_counter()
     original = normalize_whitespace(text or "")
     if not original:
         raise ValueError("Paste some text first.")
 
-    # Clean the source first so ESL errors don't propagate through the rewrite
-    # (and so the MiniLM meaning guard can't re-inject them when it reverts).
     source = original
     grammar_fixed_input = False
     if GRAMMAR_FIX_INPUT:
@@ -157,144 +203,133 @@ def rewrite_text(
 
     strength = max(0, min(2, int(strength)))
     tone = normalize_tone(tone)
-    want_gen = bool(ml_polish) and generative_available()
+    want_gen = bool(ml_polish) and GENERATIVE_PRIMARY and generative_available()
     want_minilm = bool(ml_polish) and minilm_available()
-    # When generative is the main path, skip MiniLM synonym ranking (it prefers near-copies).
-    # Still enable context for meaning guard helpers that check the flag.
+    # Skip MiniLM synonym ranking on gen path (it prefers near-copies)
     set_ml_polish(want_minilm and not want_gen)
     rng = _rng_for(
-        source
-        + "|"
-        + tone
-        + "|"
-        + str(strength)
-        + ("|ml" if (want_gen or want_minilm) else "")
+        source + "|" + tone + "|" + str(strength) + ("|ml" if (want_gen or want_minilm) else "")
     )
 
     notes = ""
     used_gen_main = False
+    rewritten = source
 
     if want_gen:
-        logger.info("ML polish: aggressive generative rewrite (%s)…", tone)
-        # Force at least Normal generative aggression
+        logger.info(
+            "Quality path: generative rewrite (%s, model=%s)…",
+            tone,
+            GENERATIVE_MODEL,
+        )
         draft = generative_paraphrase(
             source,
             tone=tone,
             strength=max(1, strength),
-            max_sim=0.86,
+            min_meaning=0.72,
         )
-        if draft and draft.strip():
-            working = tidy(draft)
+        if draft and draft.strip() and _accept_generative(source, draft):
             mark_gen_used()
             used_gen_main = True
-            # Light register polish only — do not run full synonym/structure stack
-            rewritten = _light_polish_after_gen(working, tone, strength, rng)
-            logger.info(
-                "ML polish: generative main path words %s→%s sim=%.3f",
-                word_count(source),
-                word_count(rewritten),
-                _similarity_ratio(source, rewritten),
-            )
-            # If still too similar, one more generative pass at Heavy
-            if _similarity_ratio(source, rewritten) > 0.88:
-                logger.info("ML polish: generative second pass (still too similar)")
+            rewritten = _light_post_gen(tidy(draft), tone)
+            # If still nearly identical, one heavier generative pass
+            if _similarity_ratio(source, rewritten) > 0.92:
                 draft2 = generative_paraphrase(
                     source,
                     tone=tone,
                     strength=2,
-                    max_sim=0.82,
+                    min_meaning=0.70,
                 )
-                if draft2 and draft2.strip():
-                    rewritten = _light_polish_after_gen(tidy(draft2), tone, strength, rng)
+                if draft2 and draft2.strip() and _accept_generative(source, draft2):
+                    rewritten = _light_post_gen(tidy(draft2), tone)
+            logger.info(
+                "Generative main path words %s→%s sim=%.3f",
+                word_count(source),
+                word_count(rewritten),
+                _similarity_ratio(source, rewritten),
+            )
         else:
-            tip = "Generative polish produced no draft; falling back to full rules."
+            tip = (
+                "Generative rewrite unavailable or failed validation; "
+                "using classical fallback."
+                if draft
+                else "Generative model produced no draft; using classical fallback."
+            )
             notes = tip
             used_gen_main = False
 
     if not used_gen_main:
-        # Full classical path (also when ML polish off, or gen missing)
         if want_minilm:
             set_ml_polish(True)
-        paragraphs = split_paragraphs(source)
-        out_paras = [_rewrite_paragraph(p, tone, strength, rng) for p in paragraphs]
-        rewritten = tidy("\n\n".join(out_paras))
-        rewritten = scrub_phrases(rewritten)
-        rewritten = apply_tone_style(rewritten, tone, strength, rng)
-        rewritten = apply_tone_contractions(rewritten, tone)
-
-    # Meaning guard: looser when generative wrote the draft.
-    # NOTE: guard against `source` (grammar-corrected) — so any revert restores
-    # the corrected sentence, never the original ESL error.
-    if want_minilm:
-        set_ml_polish(True)  # enable guard
-        if used_gen_main:
-            rewritten = apply_minilm_polish(
-                source, rewritten, tone, min_meaning=0.58
-            )
+        if LEXICON_FALLBACK or not ml_polish:
+            rewritten = _classical_rewrite(source, tone, strength, rng)
         else:
+            # ML requested but gen failed and lexicon disabled → corrected source only
+            rewritten = source
+            tip = "No generative output and lexicon fallback disabled; returning grammar-corrected source."
+            notes = f"{notes} {tip}".strip() if notes else tip
+
+        # Meaning guard on classical path only
+        if want_minilm:
+            set_ml_polish(True)
             rewritten = apply_minilm_polish(source, rewritten, tone)
-        rewritten = tidy(rewritten)
-        rewritten = apply_tone_style(rewritten, tone, strength, rng)
-        rewritten = apply_tone_contractions(rewritten, tone)
+            rewritten = tidy(rewritten)
+            rewritten = apply_tone_contractions(rewritten, tone)
+
+        ratio_tmp = _similarity_ratio(source, rewritten)
+        if ratio_tmp > 0.88 and LEXICON_FALLBACK:
+            logger.info("Classical near-copy (%.3f); stronger lexicon pass", ratio_tmp)
+            rewritten = _classical_rewrite(source, tone, strength, rng, force=True)
+            if want_minilm:
+                rewritten = apply_minilm_polish(source, rewritten, tone)
+                rewritten = apply_tone_contractions(rewritten, tone)
+            tip = "Applied stronger classical pass (first pass too similar)."
+            notes = f"{notes} {tip}".strip() if notes else tip
 
     rewritten = enforce_length_budget(source, rewritten, preserve_length)
     rewritten = tidy(rewritten)
 
-    ratio = _similarity_ratio(source, rewritten)
-    if ratio > 0.85 and not used_gen_main:
-        logger.info("Low change detected (ratio=%.3f); running stronger second pass", ratio)
-        rewritten = _force_more_changes(rewritten, tone, strength, rng)
-        if want_minilm:
-            rewritten = apply_minilm_polish(source, rewritten, tone)
-            rewritten = apply_tone_style(rewritten, tone, strength, rng)
-            rewritten = apply_tone_contractions(rewritten, tone)
-        rewritten = enforce_length_budget(source, rewritten, preserve_length)
-        rewritten = tidy(rewritten)
-        tip = "Applied second pass (input was too similar after first rewrite)."
+    # Final document-level validation: if gen output flipped meaning, revert to source
+    if used_gen_main and not _accept_generative(source, rewritten):
+        logger.info("Final generative output failed validation; reverting to corrected source")
+        rewritten = source
+        tip = "Generative output failed final meaning checks; returned grammar-corrected source."
         notes = f"{notes} {tip}".strip() if notes else tip
-        ratio = _similarity_ratio(source, rewritten)
-    elif ratio > 0.90 and used_gen_main and want_gen:
-        # Last resort: rules force-pass on top of weak generative output
-        logger.info("Generative output still similar (%.3f); blending rules force pass", ratio)
-        blended = _force_more_changes(rewritten, tone, min(2, strength + 1), rng)
-        if want_minilm:
-            blended = apply_minilm_polish(source, blended, tone, min_meaning=0.58)
-        rewritten = enforce_length_budget(source, tidy(blended), preserve_length)
-        rewritten = tidy(rewritten)
-        tip = "Blended rules force-pass after generative stayed too similar."
-        notes = f"{notes} {tip}".strip() if notes else tip
-        ratio = _similarity_ratio(source, rewritten)
+        used_gen_main = False
 
-    # Final grammar cleanup on the rewrite itself (catches residual agreement slips).
     if GRAMMAR_FIX_OUTPUT:
         cleaned_out = correct_text(rewritten)
         if cleaned_out and cleaned_out.strip():
             rewritten = tidy(cleaned_out)
 
+    ratio = _similarity_ratio(source, rewritten)
     changed = rewritten.strip() != original.strip()
+
     engine_bits: list[str] = []
     if GRAMMAR_FIX_INPUT or GRAMMAR_FIX_OUTPUT:
         engine_bits.append("grammar")
     if gen_was_used():
-        engine_bits.append("flan-t5")
+        engine_bits.append("generative")
+        engine_bits.append("validate")
+        engine_bits.append("rank")
     engine_bits.append("spacy" if spacy_available() else "regex-fallback")
     if used_gen_main:
-        engine_bits.extend(["rules-light", "mechanics", "tone"])
+        engine_bits.extend(["scrub-light"])
     else:
-        engine_bits.extend(["rules", "structure", "wordnet", "mechanics", "tone"])
+        engine_bits.extend(["rules", "structure", "lexicon-fallback", "mechanics", "tone"])
     if want_minilm and ml_was_used():
         engine_bits.append("minilm")
+
     if ml_polish and not want_gen and not want_minilm:
         tip = (
             "ML polish requested but generative/MiniLM models are unavailable "
-            "on this server (install torch+transformers and/or fastembed)."
+            "(install torch+transformers and/or fastembed; set GENERATIVE_MODEL)."
         )
         notes = f"{notes} {tip}".strip() if notes else tip
     elif ml_polish and want_minilm and not want_gen:
-        tip = "ML polish: MiniLM only (generative model unavailable — deploy torch+transformers)."
+        tip = "ML polish: MiniLM guard only (generative unavailable — check GENERATIVE_MODEL)."
         notes = f"{notes} {tip}".strip() if notes else tip
     elif ml_polish and want_gen and not want_minilm:
-        tip = "ML polish: generative main path (MiniLM unavailable for meaning guard)."
+        tip = "Quality path: generative + validation (MiniLM optional for scoring)."
         notes = f"{notes} {tip}".strip() if notes else tip
     if grammar_fixed_input:
         tip = "Corrected grammar in the source before rewriting."
@@ -305,23 +340,22 @@ def rewrite_text(
         f"rewrite engine={engine} strength={strength} tone={tone} "
         f"words={word_count(original)}->{word_count(rewritten)} "
         f"changed={changed} similarity={ratio:.3f} "
-        f"ml_polish={bool(ml_polish)} gen={want_gen} gen_main={used_gen_main} minilm={want_minilm}"
+        f"ml_polish={bool(ml_polish)} gen={want_gen} gen_main={used_gen_main} "
+        f"model={GENERATIVE_MODEL if want_gen else '-'}"
     )
     logger.info(msg)
     print(f"[plainrewrite] {msg}", flush=True)
 
-    if ratio > 0.82:
-        tip = (
-            "Only small lexical shifts were available for this wording. "
-            "Try Heavy strength, another tone, or ensure FLAN-T5 is loaded on the server."
-        )
-        notes = f"{notes} {tip}".strip() if notes else tip
     if not changed:
         tip = (
-            "No rewrite produced. "
-            "Enable ML polish with generative model, or try Heavy / another tone."
+            "Output matched input after validation. "
+            "Try Heavy strength, enable ML polish with a loaded generative model, "
+            "or paste text with more rewritable content."
         )
         notes = f"{notes} {tip}".strip() if notes else tip
+    elif used_gen_main and ratio > 0.90:
+        tip = "Generative rewrite stayed close to the source (meaning-first ranking)."
+        notes = f"{notes} {tip}".strip() if notes else tip
 
     return RewriteResult(
         text=rewritten,

app/pipeline/sentence_transform.py

@@ -322,9 +322,11 @@ def front_subordinate(
     if main.lower().startswith(linker.lower()):
         return None
     body = main[0].lower() + main[1:] if main and main[0].isupper() else main
-    # Academic always prefers fronting; Formal usually; Casual often keeps clause at end
+    # Academic always prefers fronting; Formal usually; Casual/Neutral often keep clause at end
     if is_casual(tone) and rng.random() < 0.55:
         return None
+    if normalize_tone(tone) == "Neutral" and rng.random() < 0.70:
+        return None
     link = linker
     if is_academic(tone) and linker.lower() == "while":
         link = "Whilst"

app/pipeline/synonym.py

@@ -29,6 +29,10 @@ _STOP_SWAP = {
     "after", "before", "during", "until", "since", "while", "among", "between",
     "under", "over", "about", "against", "upon", "onto", "into", "through",
     "across", "around", "without", "within", "beside", "behind", "above", "below",
+    # Degree adverbs — WordNet turns very→rattling and wrecks collocations
+    "very", "really", "quite", "rather", "pretty", "fairly", "highly", "extremely",
+    # Temporal discourse — WordNet nowadays→today is pointless churn
+    "nowadays", "today", "tonight", "tomorrow", "yesterday", "now",
 }
 
 # Closed-class grammar only — no domain word lists.

requirements.txt

@@ -9,7 +9,8 @@ python-multipart>=0.0.9
 httpx>=0.27.0
 PyJWT[crypto]>=2.8.0
 python-dotenv>=1.0.0
-# ML polish: MiniLM meaning guard (ONNX) + FLAN-T5-small generative draft (lazy-loaded)
+# ML polish: MiniLM meaning scoring + seq2seq rewrite (lazy-loaded)
+# Default model is flan-t5-base (override GENERATIVE_MODEL for larger GPUs)
 fastembed>=0.4.2
 torch>=2.2.0
 transformers>=4.40.0

scripts/test_meaning_fix.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+
+from app.pipeline.grammar_fix import correct_text
+from app.pipeline.meaning_safety import polarity_safe
+from app.pipeline.orchestrator import rewrite_text
+
+# --- polarity ---
+assert polarity_safe(
+    "people live an unhealthy life",
+    "people live a poor lifestyle",
+)
+assert not polarity_safe(
+    "people live an unhealthy life",
+    "people live a healthy life",
+)
+assert not polarity_safe(
+    "people don't realize how much it affects health",
+    "people realize how much it affects health",
+)
+assert not polarity_safe(
+    "people don't realize how much it affects health",
+    "causes people to realize how much it affects their health",
+)
+print("polarity OK")
+
+# --- grammar ---
+g = correct_text(
+    "Nowadays many peoples are living unhealthy life because they don't have enough times. "
+    "Eating fast foods are becoming very common and peoples don't realizes how much it affect their health."
+)
+gl = g.lower()
+assert "peoples" not in gl
+assert "enough times" not in gl
+assert "enough time" in gl
+assert "don't realize" in gl or "do not realize" in gl
+assert "it affects" in gl
+assert "fast foods is" in gl or "fast food" in gl
+assert "an unhealthy life" in gl
+print("grammar OK:")
+print(g)
+
+# --- classical rewrite ---
+orig = (
+    "Nowadays many peoples are living unhealthy life because they don't have enough times. "
+    "Eating fast foods are becoming very common and peoples don't realizes how much it affect their health."
+)
+r = rewrite_text(orig, tone="Neutral", strength=1, preserve_length=True, ml_polish=False)
+out = r.text.lower()
+print("\nNeutral Normal:")
+print(r.text)
+assert "peoples" not in out
+assert "adequate moments" not in out
+assert "communities" not in out or "people" in out  # communities banned as people swap
+assert "rapid food" not in out
+assert "highly usual" not in out
+assert "don't realize" in out or "do not realize" in out
+assert "affects" in out
+assert "nowadays" in out  # don't churn to today
+assert "individuals" not in out  # keep people
+assert "enough time" in out
+assert "an unhealthy life" in out
+assert "very common" in out or "widespread" in out
+print("\nALL ASSERTIONS PASSED")

scripts/test_quality_pipeline.py

@@ -0,0 +1,110 @@
+"""Regression tests for quality-first rewrite pipeline (no GPU required)."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+
+from app.pipeline.candidate_ranker import pick_best_candidate, score_candidate
+from app.pipeline.candidate_validator import ValidationResult, validate_candidate
+from app.pipeline.grammar_fix import correct_text
+from app.pipeline.meaning_safety import polarity_safe
+from app.pipeline.orchestrator import rewrite_text
+
+
+def test_polarity() -> None:
+    assert polarity_safe(
+        "people live an unhealthy life",
+        "people live a poor lifestyle",
+    )
+    assert not polarity_safe(
+        "people live an unhealthy life",
+        "people live a healthy life",
+    )
+    assert not polarity_safe(
+        "people don't realize how much it affects health",
+        "causes people to realize how much it affects their health",
+    )
+    print("polarity OK")
+
+
+def test_validator_rejects_flips() -> None:
+    orig = (
+        "Nowadays many people are living an unhealthy life because they don't have enough time. "
+        "Eating fast foods is becoming very common and people don't realize how much it affects their health."
+    )
+    bad = (
+        "Nowadays many people live a healthy life because they don't have time to live. "
+        "Eating fast food causes people to realize how much it affects their health."
+    )
+    v = validate_candidate(orig, bad, min_meaning=0.5)
+    assert not v.ok
+    assert "polarity" in v.reasons
+    print("validator rejects flips OK")
+
+
+def test_ranker_prefers_faithful() -> None:
+    orig = "Students should complete the assignment before the deadline."
+    good = "Students ought to finish the assignment ahead of the deadline."
+    bad = "Students should ignore the assignment after the deadline."
+    # Force validation results
+    picked = pick_best_candidate(
+        orig,
+        [bad, good, orig],
+        tone="Neutral",
+        min_meaning=0.5,
+        fallback=orig,
+    )
+    # bad may fail polarity/meaning; good should win or fallback
+    assert "ignore" not in picked.lower()
+    print("ranker OK:", picked)
+
+
+def test_grammar() -> None:
+    g = correct_text(
+        "Nowadays many peoples are living unhealthy life because they don't have enough times. "
+        "Eating fast foods are becoming very common and peoples don't realizes how much it affect their health."
+    )
+    gl = g.lower()
+    assert "peoples" not in gl
+    assert "enough time" in gl
+    assert "an unhealthy life" in gl
+    assert "don't realize" in gl
+    assert "it affects" in gl
+    print("grammar OK")
+
+
+def test_classical_fallback() -> None:
+    orig = (
+        "Nowadays many peoples are living unhealthy life because they don't have enough times. "
+        "Eating fast foods are becoming very common and peoples don't realizes how much it affect their health."
+    )
+    r = rewrite_text(
+        orig,
+        tone="Neutral",
+        strength=1,
+        preserve_length=True,
+        ml_polish=False,
+    )
+    out = r.text.lower()
+    assert "peoples" not in out
+    assert "adequate moments" not in out
+    assert "rapid food" not in out
+    assert "don't realize" in out or "do not realize" in out
+    assert "affects" in out
+    assert "an unhealthy life" in out
+    print("classical fallback OK:")
+    print(r.text)
+    print("engine:", r.engine)
+
+
+if __name__ == "__main__":
+    test_polarity()
+    test_validator_rejects_flips()
+    test_ranker_prefers_faithful()
+    test_grammar()
+    test_classical_fallback()
+    print("\nALL QUALITY PIPELINE TESTS PASSED")