Upload 105 files

README.md

@@ -110,7 +110,7 @@ Add Supabase secrets, then add the Space URL to Supabase Auth redirect URLs.
 
 ## Notes
 
-- With **ML polish**: generative rewrite is primary (`flan-t5-base` by default); classical lexicon is fallback. See [docs/ML_POLISH.md](docs/ML_POLISH.md).
+- With **ML polish**: hybrid pipeline (classical → validate → generative patch). Default generative profile in `.env.example` is SmolLM2 causal; FLAN-T5 seq2seq still supported. See [docs/ML_POLISH.md](docs/ML_POLISH.md), [docs/HYBRID_REWRITE_DESIGN.md](docs/HYBRID_REWRITE_DESIGN.md), [docs/SMOLLM2_PIPELINE_DESIGN.md](docs/SMOLLM2_PIPELINE_DESIGN.md).
 - Without ML polish: classical NLP (spaCy + WordNet + rules + grammar).
 - With Supabase enabled: **guest preview** on the homepage → sign up for Free → Pro later.
 - Admin: manage `profiles` / `plans` in the Supabase Table Editor.

app/config.py

@@ -63,11 +63,24 @@ LANGUAGE_TOOL_ENABLED = _lt_flag not in {"0", "false", "no", "off"} and bool(LAN
 MINILM_MODEL = (
     os.environ.get("MINILM_MODEL") or "sentence-transformers/all-MiniLM-L6-v2"
 ).strip()
-# Seq2seq rewrite model. Prefer base over small for quality; override via env.
-# Larger instruction models (7B+) need GPU — set GENERATIVE_MODEL explicitly.
+# Generative rewrite model. Seq2seq default: flan-t5-base.
+# Causal (SmolLM2): GENERATIVE_MODEL=HuggingFaceTB/SmolLM2-360M-Instruct + GENERATIVE_BACKEND=causal
 GENERATIVE_MODEL = (
     os.environ.get("GENERATIVE_MODEL") or "google/flan-t5-base"
 ).strip()
+_gen_backend_raw = (os.environ.get("GENERATIVE_BACKEND") or "seq2seq").strip().lower()
+if _gen_backend_raw in {"causal", "chat", "instruct", "decoder"}:
+    GENERATIVE_BACKEND = "causal"
+else:
+    GENERATIVE_BACKEND = "seq2seq"
+# Pipeline: hybrid | generative | classical
+_pipe_raw = (os.environ.get("PIPELINE_MODE") or "hybrid").strip().lower()
+if _pipe_raw in {"generative", "gen", "llm"}:
+    PIPELINE_MODE = "generative"
+elif _pipe_raw in {"classical", "rules", "lexicon"}:
+    PIPELINE_MODE = "classical"
+else:
+    PIPELINE_MODE = "hybrid"
 # 0 = rewrite entire document (no paragraph cap). Positive = safety cap.
 _gen_max_raw = (
     os.environ.get("GENERATIVE_MAX_PARAGRAPHS")

app/main.py

@@ -29,14 +29,16 @@ from app.bootstrap import ensure_resources
 from app.config import (
     APP_TITLE,
     AUTH_ENABLED,
+    GENERATIVE_BACKEND,
     GENERATIVE_MODEL,
     GRAMMAR_MAX_CHARS,
     LANGUAGE_TOOL_LANGUAGE,
     LANGUAGE_TOOL_URL,
     MAX_CHARS,
     ML_POLISH_AVAILABLE_DEFAULT,
+    PIPELINE_MODE,
 )
-from app.pipeline.generative import generative_package_present
+from app.pipeline.generative import backend_kind, generative_package_present
 from app.pipeline.grammar import (
     check_grammar,
     languagetool_reachable,
@@ -106,7 +108,8 @@ def health():
     if ml_pkg:
         engines.append("minilm")
     if gen_pkg:
-        engines.append("flan-t5")
+        engines.append("generative")
+        engines.append(GENERATIVE_BACKEND)
     return {
         "status": "ok",
         "app": APP_TITLE,
@@ -124,14 +127,18 @@ def health():
             "available": bool(ml_pkg or gen_pkg),
             "minilm": ml_pkg,
             "generative": gen_pkg,
+            "pipeline_mode": PIPELINE_MODE,
             "backend": {
                 "minilm": "fastembed|sentence-transformers" if ml_pkg else None,
                 "generative": GENERATIVE_MODEL if gen_pkg else None,
+                "generative_backend": GENERATIVE_BACKEND if gen_pkg else None,
+                "kind": backend_kind() if gen_pkg else None,
             },
             "pipeline": [
                 "grammar",
-                "generative paraphrase",
-                "validate/rank",
+                "classical baseline",
+                "align/validate",
+                "generative patch (seq2seq|causal)",
                 "rules-light",
                 "minilm meaning guard",
                 "lexicon-fallback",
@@ -208,6 +215,21 @@ def api_rewrite(
             "changed": result.changed,
             "notes": result.notes,
             "ml_polish": use_ml,
+            "pipeline_mode": result.pipeline_mode,
+            "generative_backend": GENERATIVE_BACKEND if use_ml else None,
+            "generative_model": GENERATIVE_MODEL if use_ml else None,
+            "hybrid": (
+                {
+                    "units": result.hybrid.units,
+                    "classical_kept": result.hybrid.classical_kept,
+                    "regenerated": result.hybrid.regenerated,
+                    "gen_accepted": result.hybrid.gen_accepted,
+                    "reverted_source": result.hybrid.reverted_source,
+                    "reasons": result.hybrid.reasons,
+                }
+                if result.hybrid
+                else None
+            ),
         },
         "account": account_payload(account),
     }

app/pipeline/alignment.py

@@ -0,0 +1,180 @@
+"""Paragraph-preserving sentence alignment for hybrid rewrite."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from difflib import SequenceMatcher
+
+from app.pipeline.nlp import get_nlp
+from app.pipeline.normalize import split_paragraphs, split_sentences_regex
+
+
+@dataclass
+class AlignedUnit:
+    """One aligned source/candidate span within a paragraph."""
+
+    source: str
+    candidate: str
+    paragraph_index: int
+    confidence: float
+    kind: str = "1:1"  # 1:1 | 2:1 | 1:2 | para
+
+
+def _split_sentences(text: str) -> list[str]:
+    text = (text or "").strip()
+    if not text:
+        return []
+    nlp = get_nlp()
+    if nlp is not None:
+        return [s.text.strip() for s in nlp(text).sents if s.text.strip()]
+    return split_sentences_regex(text)
+
+
+def _sim(a: str, b: str) -> float:
+    return SequenceMatcher(None, a.lower().strip(), b.lower().strip()).ratio()
+
+
+def align_paragraph_sentences(
+    source_para: str,
+    candidate_para: str,
+    *,
+    paragraph_index: int = 0,
+    min_confidence: float = 0.35,
+) -> list[AlignedUnit]:
+    """Monotonic alignment of sentences within one paragraph."""
+    src = _split_sentences(source_para)
+    cand = _split_sentences(candidate_para)
+    if not src and not cand:
+        return []
+    if not src:
+        return [
+            AlignedUnit("", c, paragraph_index, 0.0, "1:1")
+            for c in cand
+        ]
+    if not cand:
+        return [
+            AlignedUnit(s, "", paragraph_index, 0.0, "1:1")
+            for s in src
+        ]
+
+    # Equal counts → order-based pairs (verify confidence)
+    if len(src) == len(cand):
+        units: list[AlignedUnit] = []
+        for s, c in zip(src, cand):
+            conf = _sim(s, c)
+            if conf < min_confidence:
+                # Fall back to whole-paragraph unit when pairing looks wrong
+                return [
+                    AlignedUnit(
+                        source_para.strip(),
+                        candidate_para.strip(),
+                        paragraph_index,
+                        _sim(source_para, candidate_para),
+                        "para",
+                    )
+                ]
+            units.append(AlignedUnit(s, c, paragraph_index, conf, "1:1"))
+        return units
+
+    # Greedy monotonic merge for unequal counts
+    units = []
+    i = j = 0
+    while i < len(src) and j < len(cand):
+        one = _sim(src[i], cand[j])
+        merge_src = (
+            _sim(" ".join(src[i : i + 2]), cand[j]) if i + 1 < len(src) else -1.0
+        )
+        merge_cand = (
+            _sim(src[i], " ".join(cand[j : j + 2])) if j + 1 < len(cand) else -1.0
+        )
+        best = max(one, merge_src, merge_cand)
+        if best < min_confidence:
+            return [
+                AlignedUnit(
+                    source_para.strip(),
+                    candidate_para.strip(),
+                    paragraph_index,
+                    _sim(source_para, candidate_para),
+                    "para",
+                )
+            ]
+        if best == merge_src and merge_src >= one and merge_src >= merge_cand:
+            units.append(
+                AlignedUnit(
+                    " ".join(src[i : i + 2]),
+                    cand[j],
+                    paragraph_index,
+                    merge_src,
+                    "2:1",
+                )
+            )
+            i += 2
+            j += 1
+        elif best == merge_cand and merge_cand >= one:
+            units.append(
+                AlignedUnit(
+                    src[i],
+                    " ".join(cand[j : j + 2]),
+                    paragraph_index,
+                    merge_cand,
+                    "1:2",
+                )
+            )
+            i += 1
+            j += 2
+        else:
+            units.append(AlignedUnit(src[i], cand[j], paragraph_index, one, "1:1"))
+            i += 1
+            j += 1
+
+    # Trailing leftovers → attach to last unit or emit unpaired
+    while i < len(src):
+        if units:
+            last = units[-1]
+            units[-1] = AlignedUnit(
+                (last.source + " " + src[i]).strip(),
+                last.candidate,
+                paragraph_index,
+                _sim(last.source + " " + src[i], last.candidate),
+                "2:1",
+            )
+        else:
+            units.append(AlignedUnit(src[i], "", paragraph_index, 0.0, "1:1"))
+        i += 1
+    while j < len(cand):
+        if units:
+            last = units[-1]
+            units[-1] = AlignedUnit(
+                last.source,
+                (last.candidate + " " + cand[j]).strip(),
+                paragraph_index,
+                _sim(last.source, last.candidate + " " + cand[j]),
+                "1:2",
+            )
+        else:
+            units.append(AlignedUnit("", cand[j], paragraph_index, 0.0, "1:1"))
+        j += 1
+    return units
+
+
+def align_documents(source: str, candidate: str) -> list[AlignedUnit]:
+    """Align source and candidate paragraph-by-paragraph."""
+    s_paras = split_paragraphs(source)
+    c_paras = split_paragraphs(candidate)
+    # Pad shorter side so paragraphs stay in order
+    n = max(len(s_paras), len(c_paras))
+    while len(s_paras) < n:
+        s_paras.append("")
+    while len(c_paras) < n:
+        c_paras.append("")
+    out: list[AlignedUnit] = []
+    for idx, (sp, cp) in enumerate(zip(s_paras, c_paras)):
+        if not sp.strip() and not cp.strip():
+            continue
+        if not sp.strip() or not cp.strip():
+            out.append(
+                AlignedUnit(sp.strip(), cp.strip(), idx, 0.0, "para")
+            )
+            continue
+        out.extend(align_paragraph_sentences(sp, cp, paragraph_index=idx))
+    return out

app/pipeline/generative.py

@@ -1,7 +1,7 @@
-"""Paragraph-level generative rewrite (quality-first path).
+"""Generative rewrite: seq2seq (FLAN-T5) or causal chat (SmolLM2).
 
 Pipeline when models load:
-  1) Draft several paraphrases per paragraph (seq2seq)
+  1) Draft several paraphrases per unit
   2) Validate polarity / facts / meaning (reject near-copies)
   3) Rank survivors for fidelity + useful difference
 
@@ -17,6 +17,7 @@ from difflib import SequenceMatcher
 from typing import Any
 
 from app.config import (
+    GENERATIVE_BACKEND,
     GENERATIVE_MAX_PARAGRAPHS,
     GENERATIVE_MODEL,
     GENERATIVE_POLISH_ENABLED,
@@ -33,6 +34,14 @@ _tokenizer: Any = None
 _model: Any = None
 _failed = False
 _backend: str | None = None
+_backend_kind: str | None = None  # seq2seq | causal
+
+_SYSTEM_REWRITE = (
+    "You rewrite English text. Keep EVERY claim and the same polarity. "
+    "Do not flip negatives or antonyms. Preserve names, numbers, dates, and quotations. "
+    "Use different wording and sentence structure — do not copy phrases almost verbatim. "
+    "Do not summarize, drop, or invent content. Return only the rewritten text."
+)
 
 _SHARED_RULES = (
     "Keep ALL sentences and claims from the paragraph — do not drop or merge away ideas.\n"
@@ -80,6 +89,20 @@ _RETRY_PROMPT = (
     "Paraphrase:"
 )
 
+_CAUSAL_USER = (
+    "Tone: {tone}.\n"
+    "Rewrite the following text. Keep every claim and polarity. "
+    "Do not drop or invent content.\n\n"
+    "{text}"
+)
+
+_CAUSAL_RETRY_USER = (
+    "Tone: {tone}.\n"
+    "Rewrite again with clearly different wording and sentence structure, "
+    "but the exact same meaning and polarity. Do not copy long phrases.\n\n"
+    "{text}"
+)
+
 
 def generative_package_present() -> bool:
     if not ML_POLISH_AVAILABLE_DEFAULT or not GENERATIVE_POLISH_ENABLED:
@@ -112,8 +135,13 @@ def backend_name() -> str | None:
     return _backend
 
 
+def backend_kind() -> str:
+    """Configured backend kind: seq2seq or causal (even before model load)."""
+    return GENERATIVE_BACKEND if GENERATIVE_BACKEND in {"seq2seq", "causal"} else "seq2seq"
+
+
 def _ensure_model():
-    global _tokenizer, _model, _failed, _backend
+    global _tokenizer, _model, _failed, _backend, _backend_kind
     if _model is not None or _failed:
         return _model
     with _lock:
@@ -121,15 +149,31 @@ def _ensure_model():
             return _model
         try:
             import torch
-            from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+            from transformers import AutoTokenizer
 
             name = GENERATIVE_MODEL
-            logger.info("Loading generative rewrite model %s …", name)
+            kind = backend_kind()
+            logger.info(
+                "Loading generative rewrite model %s (backend=%s) …", name, kind
+            )
             _tokenizer = AutoTokenizer.from_pretrained(name)
-            _model = AutoModelForSeq2SeqLM.from_pretrained(name)
+            if _tokenizer.pad_token is None and _tokenizer.eos_token is not None:
+                _tokenizer.pad_token = _tokenizer.eos_token
+
+            if kind == "causal":
+                from transformers import AutoModelForCausalLM
+
+                _model = AutoModelForCausalLM.from_pretrained(name)
+                _backend_kind = "causal"
+            else:
+                from transformers import AutoModelForSeq2SeqLM
+
+                _model = AutoModelForSeq2SeqLM.from_pretrained(name)
+                _backend_kind = "seq2seq"
+
             _model.eval()
             _model.to("cpu")
-            _backend = f"transformers:{name}"
+            _backend = f"transformers:{kind}:{name}"
             _ = torch.__version__
             logger.info("Generative rewrite ready (%s)", _backend)
             return _model
@@ -139,6 +183,7 @@ def _ensure_model():
             _tokenizer = None
             _model = None
             _backend = None
+            _backend_kind = None
             return None
 
 
@@ -146,11 +191,18 @@ def _clean_gen_text(text: str) -> str:
     text = re.sub(r"\s+", " ", (text or "").strip())
     if not text:
         return ""
+    # Drop casual chat wrappers
+    text = re.sub(
+        r"^(?:assistant|system|user)\s*[:\-]\s*", "", text, flags=re.I
+    ).strip()
     low = text.lower()
     for prefix in (
         "rewrite:",
         "paraphrase:",
         "new paraphrase:",
+        "rewritten text:",
+        "here is the rewrite:",
+        "here is the rewritten text:",
         "input:",
         "paragraph:",
         "sentence:",
@@ -159,6 +211,9 @@ def _clean_gen_text(text: str) -> str:
         if low.startswith(prefix):
             text = text[len(prefix) :].strip()
             low = text.lower()
+    # Strip surrounding quotes if the whole output is quoted
+    if len(text) >= 2 and text[0] in "\"'" and text[-1] == text[0]:
+        text = text[1:-1].strip()
     return text
 
 
@@ -182,6 +237,9 @@ def _gen_kwargs(strength: int, *, n_return: int | None = None) -> dict[str, Any]
         2: {"temperature": 1.05, "top_p": 0.95, "top_k": 60, "max_new_tokens": 320, "n": 5},
     }[strength]
     n = n_return if n_return is not None else base["n"]
+    # Causal models: fewer return sequences to limit CPU RAM
+    if _backend_kind == "causal":
+        n = min(int(n), 3)
     return {
         "do_sample": True,
         "num_beams": 1,
@@ -190,27 +248,92 @@ def _gen_kwargs(strength: int, *, n_return: int | None = None) -> dict[str, Any]
         "top_k": base["top_k"],
         "max_new_tokens": base["max_new_tokens"],
         "no_repeat_ngram_size": 3,
-        "num_return_sequences": max(3, min(int(n), 5)),
+        "num_return_sequences": max(2, min(int(n), 5)),
     }
 
 
-def _generate_once(prompt: str, strength: int, *, n_return: int | None = None) -> list[str]:
+def _build_causal_prompt(text: str, tone: str, *, retry: bool = False) -> str:
+    tone_l = normalize_tone(tone)
+    user = (_CAUSAL_RETRY_USER if retry else _CAUSAL_USER).format(
+        tone=tone_l, text=text.strip()
+    )
+    messages = [
+        {"role": "system", "content": _SYSTEM_REWRITE},
+        {"role": "user", "content": user},
+    ]
+    assert _tokenizer is not None
+    try:
+        return _tokenizer.apply_chat_template(
+            messages,
+            tokenize=False,
+            add_generation_prompt=True,
+        )
+    except Exception:
+        # Fallback if chat template missing
+        return (
+            f"System: {_SYSTEM_REWRITE}\n\n"
+            f"User: {user}\n\n"
+            f"Assistant:"
+        )
+
+
+def _generate_seq2seq(prompt: str, strength: int, *, n_return: int | None = None) -> list[str]:
+    import torch
+
+    assert _model is not None and _tokenizer is not None
+    inputs = _tokenizer(
+        prompt,
+        return_tensors="pt",
+        truncation=True,
+        max_length=512,
+    )
+    kwargs = _gen_kwargs(strength, n_return=n_return)
+    with torch.no_grad():
+        out_ids = _model.generate(**inputs, **kwargs)
+    return [_tokenizer.decode(row, skip_special_tokens=True) for row in out_ids]
+
+
+def _generate_causal(prompt: str, strength: int, *, n_return: int | None = None) -> list[str]:
+    import torch
+
+    assert _model is not None and _tokenizer is not None
+    inputs = _tokenizer(
+        prompt,
+        return_tensors="pt",
+        truncation=True,
+        max_length=1024,
+    )
+    input_len = int(inputs["input_ids"].shape[-1])
+    kwargs = _gen_kwargs(strength, n_return=n_return)
+    # Avoid pad/eos issues on small causal models
+    if _tokenizer.eos_token_id is not None:
+        kwargs["eos_token_id"] = _tokenizer.eos_token_id
+    if _tokenizer.pad_token_id is not None:
+        kwargs["pad_token_id"] = _tokenizer.pad_token_id
+    with torch.no_grad():
+        out_ids = _model.generate(**inputs, **kwargs)
+    decoded: list[str] = []
+    for row in out_ids:
+        new_tokens = row[input_len:]
+        decoded.append(_tokenizer.decode(new_tokens, skip_special_tokens=True))
+    return decoded
+
+
+def _generate_once(
+    prompt: str,
+    strength: int,
+    *,
+    n_return: int | None = None,
+    causal: bool | None = None,
+) -> list[str]:
     model = _ensure_model()
     if model is None or _tokenizer is None:
         return []
+    use_causal = (_backend_kind == "causal") if causal is None else causal
     try:
-        import torch
-
-        inputs = _tokenizer(
-            prompt,
-            return_tensors="pt",
-            truncation=True,
-            max_length=512,
-        )
-        kwargs = _gen_kwargs(strength, n_return=n_return)
-        with torch.no_grad():
-            out_ids = model.generate(**inputs, **kwargs)
-        return [_tokenizer.decode(row, skip_special_tokens=True) for row in out_ids]
+        if use_causal:
+            return _generate_causal(prompt, strength, n_return=n_return)
+        return _generate_seq2seq(prompt, strength, n_return=n_return)
     except Exception as exc:
         logger.warning("Generative decode failed: %s", exc)
         return []
@@ -239,6 +362,29 @@ def _chunk_long_paragraph(para: str, max_words: int = 180) -> list[str]:
     return chunks or [para]
 
 
+def rewrite_unit(
+    text: str,
+    *,
+    tone: str = "Neutral",
+    strength: int = 1,
+    min_meaning: float = 0.72,
+    max_sim: float = NEAR_COPY_SURFACE,
+) -> str:
+    """
+    Public targeted-unit rewrite (sentence or short paragraph).
+
+    Backend-agnostic: uses seq2seq or causal depending on GENERATIVE_BACKEND.
+    Returns the best validated rewrite, or the original unit if generation fails.
+    """
+    return _paraphrase_paragraph(
+        text,
+        tone,
+        strength,
+        min_meaning=min_meaning,
+        max_sim=max_sim,
+    )
+
+
 def _paraphrase_paragraph(
     paragraph: str,
     tone: str,
@@ -247,7 +393,7 @@ def _paraphrase_paragraph(
     min_meaning: float = 0.72,
     max_sim: float = NEAR_COPY_SURFACE,
 ) -> str:
-    """Generate candidates for one paragraph; return best validated rewrite or original."""
+    """Generate candidates for one unit; return best validated rewrite or original."""
     model = _ensure_model()
     if model is None or _tokenizer is None:
         return paragraph
@@ -255,7 +401,12 @@ def _paraphrase_paragraph(
         return paragraph
 
     tone_l = normalize_tone(tone)
-    prompt = _TONE_PROMPTS.get(tone_l, _TONE_PROMPTS["Neutral"]).format(text=paragraph.strip())
+    if _backend_kind == "causal":
+        prompt = _build_causal_prompt(paragraph, tone_l, retry=False)
+    else:
+        prompt = _TONE_PROMPTS.get(tone_l, _TONE_PROMPTS["Neutral"]).format(
+            text=paragraph.strip()
+        )
     raw = _generate_once(prompt, strength)
     cands = [_clean_gen_text(x) for x in raw]
 
@@ -268,10 +419,14 @@ def _paraphrase_paragraph(
         fallback=paragraph,
     )
 
-    # Retry when only near-copies / identical survived
     if _is_near_copy(paragraph, picked, max_sim):
-        retry = _RETRY_PROMPT.format(tone=tone_l.lower(), text=paragraph.strip())
-        raw2 = _generate_once(retry, min(2, strength + 1), n_return=5)
+        if _backend_kind == "causal":
+            retry_prompt = _build_causal_prompt(paragraph, tone_l, retry=True)
+        else:
+            retry_prompt = _RETRY_PROMPT.format(
+                tone=tone_l.lower(), text=paragraph.strip()
+            )
+        raw2 = _generate_once(retry_prompt, min(2, strength + 1), n_return=5)
         cands2 = [_clean_gen_text(x) for x in raw2]
         picked2 = pick_best_candidate(
             paragraph,
@@ -285,7 +440,7 @@ def _paraphrase_paragraph(
             picked = picked2
         else:
             logger.info(
-                "Generative paragraph still near-copy (sim=%.3f); keeping source chunk",
+                "Generative unit still near-copy (sim=%.3f); keeping source chunk",
                 _surface_sim(paragraph, picked2),
             )
             return paragraph
@@ -348,15 +503,16 @@ def generative_paraphrase(
         return None
     result = "\n\n".join(p for p in out_paras if p)
     logger.info(
-        "Generative paraphrase: %s/%s units changed (strength=%s, model=%s, budget=%s, max_sim=%.2f)",
+        "Generative paraphrase: %s/%s units changed (strength=%s, model=%s, "
+        "backend=%s, budget=%s, max_sim=%.2f)",
         changed_n,
         used,
         gen_strength,
         GENERATIVE_MODEL,
+        _backend_kind or backend_kind(),
         "all" if unlimited else budget,
         max_sim,
     )
-    # Whole-doc still a near-copy → signal orchestrator to use classical fallback
     if used > 0 and changed_n == 0:
         logger.info("Generative produced no useful paraphrases; returning near-copy for fallback")
     return result
@@ -366,10 +522,10 @@ def warm_generative() -> bool:
     try:
         ok = generative_available()
         if ok:
-            _paraphrase_paragraph(
+            rewrite_unit(
                 "The system works well for most users and saves time.",
-                "Neutral",
-                1,
+                tone="Neutral",
+                strength=1,
             )
         return ok
     except Exception as exc:

app/pipeline/orchestrator.py

@@ -1,7 +1,9 @@
-"""Orchestrate quality-first rewrite: grammar → generative → validate → grammar.
+"""Orchestrate rewrite: classical / generative / hybrid (SmolLM2 or FLAN patch).
 
-Classical lexicon/WordNet is demoted to offline fallback when generative is
-unavailable or ML polish is off.
+Modes (PIPELINE_MODE):
+  classical  — rules + lexicon only
+  generative — whole-document generative rewrite
+  hybrid     — classical baseline → validate units → generative patch failed units
 """
 
 from __future__ import annotations
@@ -9,18 +11,27 @@ from __future__ import annotations
 import logging
 import random
 import time
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from difflib import SequenceMatcher
 
 from app.config import (
+    GENERATIVE_BACKEND,
     GENERATIVE_MODEL,
     GENERATIVE_PRIMARY,
     GRAMMAR_FIX_INPUT,
     GRAMMAR_FIX_OUTPUT,
     LEXICON_FALLBACK,
+    PIPELINE_MODE,
 )
+from app.pipeline.alignment import align_documents
+from app.pipeline.candidate_ranker import NEAR_COPY_SURFACE
 from app.pipeline.candidate_validator import validate_candidate
-from app.pipeline.generative import generative_available, generative_paraphrase
+from app.pipeline.generative import (
+    backend_kind,
+    generative_available,
+    generative_paraphrase,
+    rewrite_unit,
+)
 from app.pipeline.grammar_fix import correct_text
 from app.pipeline.mechanics import enforce_length_budget, scrub_phrases, tidy
 from app.pipeline.ml_context import (
@@ -52,6 +63,20 @@ if not logger.handlers:
     logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
 
 
+@dataclass
+class HybridStats:
+    units: int = 0
+    classical_kept: int = 0
+    regenerated: int = 0
+    gen_accepted: int = 0
+    reverted_source: int = 0
+    reasons: dict[str, int] = field(default_factory=dict)
+
+    def bump_reason(self, reason: str) -> None:
+        key = reason.split(":")[0] if reason else "other"
+        self.reasons[key] = self.reasons.get(key, 0) + 1
+
+
 @dataclass
 class RewriteResult:
     text: str
@@ -63,6 +88,8 @@ class RewriteResult:
     tone: str
     changed: bool
     notes: str = ""
+    pipeline_mode: str = ""
+    hybrid: HybridStats | None = None
 
 
 def _rng_for(text: str) -> random.Random:
@@ -114,7 +141,6 @@ def _classical_rewrite(
     rewritten = apply_tone_style(rewritten, tone, strength, rng)
     rewritten = apply_tone_contractions(rewritten, tone)
     if force:
-        # Second-pass lexicon force for stubborn near-copies
         bump = min(2, strength + 1)
         paras = split_paragraphs(rewritten)
         forced: list[str] = []
@@ -139,19 +165,33 @@ def _classical_rewrite(
     return rewritten
 
 
+def _hard_fail(reasons: list[str]) -> bool:
+    for r in reasons:
+        if r in {"polarity", "numbers", "quotes", "length", "coverage"}:
+            return True
+        if r.startswith("meaning") or r.startswith("entity"):
+            return True
+    return False
+
+
+def _needs_patch(source_unit: str, candidate: str, *, max_sim: float = NEAR_COPY_SURFACE) -> tuple[bool, list[str]]:
+    """Return (needs_regen, reasons). Near-copy or validation failure → patch."""
+    if not candidate.strip():
+        return True, ["empty"]
+    if not source_unit.strip():
+        return False, []
+    v = validate_candidate(source_unit, candidate, min_meaning=0.72, max_surface=max_sim)
+    if v.ok:
+        return False, []
+    # identical / too_similar → patch; hard fails → patch; soft → still patch if not ok
+    return True, list(v.reasons)
+
+
 def _accept_generative(source: str, draft: str) -> bool:
     """Whole-document gate: polarity/meaning must hold vs grammar-corrected source."""
     if not draft or not draft.strip():
         return False
 
-    def _hard_fail(reasons: list[str]) -> bool:
-        for r in reasons:
-            if r in {"polarity", "numbers", "quotes", "length", "coverage"}:
-                return True
-            if r.startswith("meaning") or r.startswith("entity"):
-                return True
-        return False
-
     s_paras = split_paragraphs(source)
     d_paras = split_paragraphs(draft)
     if len(s_paras) == len(d_paras) and s_paras:
@@ -172,6 +212,98 @@ def _accept_generative(source: str, draft: str) -> bool:
     return True
 
 
+def _hybrid_rewrite(
+    source: str,
+    tone: str,
+    strength: int,
+    rng: random.Random,
+    *,
+    can_generate: bool,
+) -> tuple[str, HybridStats, bool]:
+    """
+    Classical baseline → align → validate → generative patch failed units.
+
+    Returns (text, stats, used_any_gen).
+    """
+    stats = HybridStats()
+    baseline = _classical_rewrite(source, tone, strength, rng)
+    units = align_documents(source, baseline)
+    if not units:
+        return baseline, stats, False
+
+    # Group by paragraph to reassemble with blank lines
+    by_para: dict[int, list[str]] = {}
+    used_gen = False
+
+    for unit in units:
+        stats.units += 1
+        src_u = unit.source.strip()
+        cand_u = unit.candidate.strip()
+        if not src_u:
+            if cand_u:
+                by_para.setdefault(unit.paragraph_index, []).append(cand_u)
+            continue
+
+        needs, reasons = _needs_patch(src_u, cand_u)
+        if not needs:
+            stats.classical_kept += 1
+            by_para.setdefault(unit.paragraph_index, []).append(cand_u or src_u)
+            continue
+
+        for r in reasons:
+            stats.bump_reason(r)
+
+        if not can_generate:
+            # Prefer classical if soft failure; revert to source on hard fail / empty
+            if cand_u and not _hard_fail(reasons) and "too_similar" not in reasons and "identical" not in reasons:
+                stats.classical_kept += 1
+                by_para.setdefault(unit.paragraph_index, []).append(cand_u)
+            else:
+                # Near-copy with no gen: keep classical if present else source
+                if cand_u and ("too_similar" in reasons or "identical" in reasons):
+                    stats.classical_kept += 1
+                    by_para.setdefault(unit.paragraph_index, []).append(cand_u)
+                else:
+                    stats.reverted_source += 1
+                    by_para.setdefault(unit.paragraph_index, []).append(src_u)
+            continue
+
+        stats.regenerated += 1
+        mark_gen_used()
+        used_gen = True
+        gen_raw = rewrite_unit(
+            src_u,
+            tone=tone,
+            strength=max(1, strength),
+            min_meaning=0.72,
+            max_sim=0.92,
+        )
+        # Grammar-clean generated unit before revalidation
+        gen_clean = correct_text(gen_raw) if gen_raw else gen_raw
+        gen_clean = tidy(gen_clean or "")
+
+        vg = validate_candidate(src_u, gen_clean, min_meaning=0.70, max_surface=0.92)
+        near = _similarity_ratio(src_u, gen_clean) >= 0.92
+        if vg.ok and not near:
+            stats.gen_accepted += 1
+            by_para.setdefault(unit.paragraph_index, []).append(gen_clean)
+        else:
+            stats.reverted_source += 1
+            for r in vg.reasons:
+                stats.bump_reason(r)
+            logger.info(
+                "Hybrid gen unit rejected (%s); keeping corrected source",
+                vg.reasons or ["near_copy"],
+            )
+            by_para.setdefault(unit.paragraph_index, []).append(src_u)
+
+    para_idxs = sorted(by_para.keys())
+    paras_out = [" ".join(by_para[i]).strip() for i in para_idxs if by_para[i]]
+    text = tidy("\n\n".join(p for p in paras_out if p))
+    text = _light_post_gen(text, tone)
+    return text, stats, used_gen
+
+
 def rewrite_text(
     text: str,
     *,
@@ -183,10 +315,9 @@ def rewrite_text(
     """
     Quality-first rewrite.
 
-    When ml_polish=True and generative model loads:
-      grammar → paragraph generative (multi-candidate validate/rank) → light scrub → grammar
-    Otherwise (or if gen fails validation):
-      grammar → classical lexicon fallback (if LEXICON_FALLBACK) → grammar
+    PIPELINE_MODE=hybrid (default with ML): classical → validate → generative patch
+    PIPELINE_MODE=generative: whole-document generative rewrite
+    PIPELINE_MODE=classical / ml_polish off: classical lexicon path
     """
     started = time.perf_counter()
     original = normalize_whitespace(text or "")
@@ -203,23 +334,61 @@ def rewrite_text(
 
     strength = max(0, min(2, int(strength)))
     tone = normalize_tone(tone)
-    want_gen = bool(ml_polish) and GENERATIVE_PRIMARY and generative_available()
+    mode = PIPELINE_MODE if ml_polish else "classical"
+    if mode == "classical" and not ml_polish:
+        pass
+    elif not ml_polish:
+        mode = "classical"
+    elif mode == "hybrid" and not GENERATIVE_PRIMARY:
+        # hybrid without gen primary still runs classical+validate; patch only if model loads
+        pass
+
+    gen_ready = bool(ml_polish) and generative_available()
     want_minilm = bool(ml_polish) and minilm_available()
-    # Skip MiniLM synonym ranking on gen path (it prefers near-copies)
-    set_ml_polish(want_minilm and not want_gen)
+    # Skip MiniLM synonym ranking on generative-heavy paths
+    set_ml_polish(want_minilm and mode == "classical")
     rng = _rng_for(
-        source + "|" + tone + "|" + str(strength) + ("|ml" if (want_gen or want_minilm) else "")
+        source + "|" + tone + "|" + str(strength) + ("|ml" if ml_polish else "")
     )
 
     notes = ""
     used_gen_main = False
+    hybrid_stats: HybridStats | None = None
     rewritten = source
 
-    if want_gen:
+    if mode == "hybrid" and ml_polish:
         logger.info(
-            "Quality path: generative rewrite (%s, model=%s)…",
+            "Hybrid path: classical → validate → patch (%s, model=%s, backend=%s)…",
+            tone,
+            GENERATIVE_MODEL if gen_ready else "-",
+            GENERATIVE_BACKEND if gen_ready else "-",
+        )
+        rewritten, hybrid_stats, used_gen_main = _hybrid_rewrite(
+            source,
+            tone,
+            strength,
+            rng,
+            can_generate=gen_ready,
+        )
+        if not gen_ready:
+            tip = (
+                "Hybrid: generative unavailable; classical baseline with safety fallbacks."
+            )
+            notes = tip
+        else:
+            tip = (
+                f"Hybrid: kept={hybrid_stats.classical_kept} "
+                f"patched={hybrid_stats.gen_accepted}/{hybrid_stats.regenerated} "
+                f"reverted={hybrid_stats.reverted_source}"
+            )
+            notes = tip
+
+    elif mode == "generative" and ml_polish and gen_ready:
+        logger.info(
+            "Generative path (%s, model=%s, backend=%s)…",
             tone,
             GENERATIVE_MODEL,
+            GENERATIVE_BACKEND,
         )
         draft = generative_paraphrase(
             source,
@@ -230,7 +399,6 @@ def rewrite_text(
         )
         if draft and draft.strip() and _accept_generative(source, draft):
             sim0 = _similarity_ratio(source, draft)
-            # Near-copy does not count as a successful generative rewrite
             if sim0 >= 0.92:
                 logger.info(
                     "Generative draft too similar (%.3f); trying stronger pass",
@@ -294,18 +462,16 @@ def rewrite_text(
             notes = tip
             used_gen_main = False
 
-    if not used_gen_main:
+    if mode != "hybrid" and not used_gen_main:
         if want_minilm:
             set_ml_polish(True)
         if LEXICON_FALLBACK or not ml_polish:
             rewritten = _classical_rewrite(source, tone, strength, rng)
         else:
-            # ML requested but gen failed and lexicon disabled → corrected source only
             rewritten = source
             tip = "No generative output and lexicon fallback disabled; returning grammar-corrected source."
             notes = f"{notes} {tip}".strip() if notes else tip
 
-        # Meaning guard on classical path only
         if want_minilm:
             set_ml_polish(True)
             rewritten = apply_minilm_polish(source, rewritten, tone)
@@ -325,8 +491,7 @@ def rewrite_text(
     rewritten = enforce_length_budget(source, rewritten, preserve_length)
     rewritten = tidy(rewritten)
 
-    # Final document-level validation: if gen output flipped meaning, revert to source
-    if used_gen_main and not _accept_generative(source, rewritten):
+    if used_gen_main and mode == "generative" and not _accept_generative(source, rewritten):
         logger.info("Final generative output failed validation; reverting to corrected source")
         rewritten = source
         tip = "Generative output failed final meaning checks; returned grammar-corrected source."
@@ -344,41 +509,46 @@ def rewrite_text(
     engine_bits: list[str] = []
     if GRAMMAR_FIX_INPUT or GRAMMAR_FIX_OUTPUT:
         engine_bits.append("grammar")
-    if gen_was_used():
-        engine_bits.append("generative")
+    if mode == "hybrid":
+        engine_bits.extend(["classical", "align", "validate"])
+        if used_gen_main or (hybrid_stats and hybrid_stats.regenerated):
+            engine_bits.append(f"generative-patch:{backend_kind()}")
+            engine_bits.append("rank")
+        engine_bits.append("spacy" if spacy_available() else "regex-fallback")
+        engine_bits.append("scrub-light")
+    elif gen_was_used():
+        engine_bits.append(f"generative:{backend_kind()}")
         engine_bits.append("validate")
         engine_bits.append("rank")
-    engine_bits.append("spacy" if spacy_available() else "regex-fallback")
-    if used_gen_main:
-        engine_bits.extend(["scrub-light"])
+        engine_bits.append("spacy" if spacy_available() else "regex-fallback")
+        engine_bits.append("scrub-light")
     else:
+        engine_bits.append("spacy" if spacy_available() else "regex-fallback")
         engine_bits.extend(["rules", "structure", "lexicon-fallback", "mechanics", "tone"])
     if want_minilm and ml_was_used():
         engine_bits.append("minilm")
 
-    if ml_polish and not want_gen and not want_minilm:
+    if ml_polish and not gen_ready and not want_minilm:
         tip = (
             "ML polish requested but generative/MiniLM models are unavailable "
             "(install torch+transformers and/or fastembed; set GENERATIVE_MODEL)."
         )
         notes = f"{notes} {tip}".strip() if notes else tip
-    elif ml_polish and want_minilm and not want_gen:
+    elif ml_polish and want_minilm and not gen_ready and mode != "hybrid":
         tip = "ML polish: MiniLM guard only (generative unavailable — check GENERATIVE_MODEL)."
         notes = f"{notes} {tip}".strip() if notes else tip
-    elif ml_polish and want_gen and not want_minilm:
-        tip = "Quality path: generative + validation (MiniLM optional for scoring)."
-        notes = f"{notes} {tip}".strip() if notes else tip
     if grammar_fixed_input:
         tip = "Corrected grammar in the source before rewriting."
         notes = f"{notes} {tip}".strip() if notes else tip
 
     engine = "+".join(engine_bits)
     msg = (
-        f"rewrite engine={engine} strength={strength} tone={tone} "
+        f"rewrite engine={engine} mode={mode} strength={strength} tone={tone} "
         f"words={word_count(original)}->{word_count(rewritten)} "
         f"changed={changed} similarity={ratio:.3f} "
-        f"ml_polish={bool(ml_polish)} gen={want_gen} gen_main={used_gen_main} "
-        f"model={GENERATIVE_MODEL if want_gen else '-'}"
+        f"ml_polish={bool(ml_polish)} gen={gen_ready} gen_main={used_gen_main} "
+        f"backend={GENERATIVE_BACKEND if gen_ready else '-'} "
+        f"model={GENERATIVE_MODEL if gen_ready else '-'}"
     )
     logger.info(msg)
     print(f"[plainrewrite] {msg}", flush=True)
@@ -391,7 +561,7 @@ def rewrite_text(
         )
         notes = f"{notes} {tip}".strip() if notes else tip
     elif used_gen_main and ratio > 0.90:
-        tip = "Generative rewrite stayed close to the source (meaning-first ranking)."
+        tip = "Rewrite stayed close to the source (meaning-first ranking)."
         notes = f"{notes} {tip}".strip() if notes else tip
 
     return RewriteResult(
@@ -404,6 +574,8 @@ def rewrite_text(
         tone=tone,
         changed=changed,
         notes=notes,
+        pipeline_mode=mode,
+        hybrid=hybrid_stats,
     )
 
 

requirements.txt

@@ -9,9 +9,10 @@ python-multipart>=0.0.9
 httpx>=0.27.0
 PyJWT[crypto]>=2.8.0
 python-dotenv>=1.0.0
-# ML polish: MiniLM meaning scoring + seq2seq rewrite (lazy-loaded)
-# Default model is flan-t5-base (override GENERATIVE_MODEL for larger GPUs)
+# ML polish: MiniLM meaning scoring + generative rewrite (lazy-loaded)
+# Backends: seq2seq (FLAN-T5) or causal chat (SmolLM2) via GENERATIVE_BACKEND
 fastembed>=0.4.2
 torch>=2.2.0
 transformers>=4.40.0
 sentencepiece>=0.2.0
+accelerate>=0.30.0

scripts/test_smollm2_pipeline.py

@@ -0,0 +1,138 @@
+"""Regression tests for SmolLM2 / hybrid pipeline (no model download required)."""
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+
+from app.config import GENERATIVE_BACKEND, PIPELINE_MODE
+from app.pipeline.alignment import align_documents, align_paragraph_sentences
+from app.pipeline.candidate_validator import validate_candidate
+from app.pipeline.generative import backend_kind, _clean_gen_text
+from app.pipeline.grammar_fix import correct_text
+from app.pipeline.orchestrator import rewrite_text
+
+
+def test_backend_defaults() -> None:
+    # Defaults come from env at import time; kind helper mirrors config
+    assert backend_kind() in {"seq2seq", "causal"}
+    assert GENERATIVE_BACKEND in {"seq2seq", "causal"}
+    assert PIPELINE_MODE in {"hybrid", "generative", "classical"}
+    print("backend defaults OK:", GENERATIVE_BACKEND, PIPELINE_MODE)
+
+
+def test_clean_causal_wrappers() -> None:
+    assert _clean_gen_text('Assistant: Hello there.') == "Hello there."
+    assert _clean_gen_text('"Rewritten sentence."') == "Rewritten sentence."
+    assert _clean_gen_text("Paraphrase: People live healthier lives.") == (
+        "People live healthier lives."
+    )
+    print("clean wrappers OK")
+
+
+def test_alignment_1to1() -> None:
+    src = "Alpha is first. Beta is second."
+    cand = "Alpha comes first. Beta comes second."
+    units = align_paragraph_sentences(src, cand)
+    assert len(units) == 2
+    assert all(u.kind == "1:1" for u in units)
+    print("alignment 1:1 OK")
+
+
+def test_alignment_preserves_paragraphs() -> None:
+    src = "One. Two.\n\nThree."
+    cand = "Uno. Dos.\n\nTres."
+    units = align_documents(src, cand)
+    idxs = {u.paragraph_index for u in units}
+    assert 0 in idxs and 1 in idxs
+    print("alignment paragraphs OK:", len(units), "units")
+
+
+def test_truncated_rejected() -> None:
+    orig = (
+        "Nowadays many people are living an unhealthy life because they don't have enough time. "
+        "Eating fast foods is becoming very common and people don't realize how much it affects their health."
+    )
+    truncated = (
+        "Fast food is becoming very common because many people don't realize "
+        "how much it affects their health."
+    )
+    v = validate_candidate(orig, truncated, min_meaning=0.5)
+    assert not v.ok
+    assert "length" in v.reasons or "coverage" in v.reasons
+    print("truncated rejected OK")
+
+
+def test_hybrid_classical_without_ml() -> None:
+    orig = (
+        "Nowadays many peoples are living unhealthy life because they don't have enough times. "
+        "Eating fast foods are becoming very common and peoples don't realizes how much it affect their health."
+    )
+    r = rewrite_text(
+        orig,
+        tone="Neutral",
+        strength=1,
+        preserve_length=True,
+        ml_polish=False,
+    )
+    out = r.text.lower()
+    assert "peoples" not in out
+    assert "an unhealthy life" in out
+    assert "affects" in out
+    assert "enough time" in out
+    # Both claims should survive classical path
+    assert "unhealthy" in out
+    assert "fast food" in out or "fast foods" in out
+    assert r.pipeline_mode == "classical"
+    print("hybrid classical-without-ml OK:", r.engine)
+
+
+def test_hybrid_mode_with_ml_no_model() -> None:
+    """When generative fails to load, hybrid still returns safe classical output."""
+    orig = (
+        "Nowadays many peoples are living unhealthy life because they don't have enough times. "
+        "Eating fast foods are becoming very common and peoples don't realizes how much it affect their health."
+    )
+    # Force unavailable generative by not requiring it; hybrid runs classical+validate
+    r = rewrite_text(
+        orig,
+        tone="Neutral",
+        strength=1,
+        preserve_length=True,
+        ml_polish=True,
+    )
+    out = r.text.lower()
+    assert "unhealthy" in out
+    assert "health" in out
+    # Must not collapse to single truncated claim only
+    g = correct_text(orig).lower()
+    assert "enough time" in out or "enough time" in g
+    print("hybrid ml path OK mode=", r.pipeline_mode, "engine=", r.engine)
+    if r.hybrid:
+        print(
+            "  units=",
+            r.hybrid.units,
+            "kept=",
+            r.hybrid.classical_kept,
+            "regen=",
+            r.hybrid.regenerated,
+            "accepted=",
+            r.hybrid.gen_accepted,
+            "reverted=",
+            r.hybrid.reverted_source,
+        )
+
+
+if __name__ == "__main__":
+    test_backend_defaults()
+    test_clean_causal_wrappers()
+    test_alignment_1to1()
+    test_alignment_preserves_paragraphs()
+    test_truncated_rejected()
+    test_hybrid_classical_without_ml()
+    test_hybrid_mode_with_ml_no_model()
+    print("\nALL SMOLLM2 / HYBRID PIPELINE TESTS PASSED")