Upload 89 files

app/engine/lexical.py

@@ -574,9 +574,21 @@ def refine_sentence(
     except Exception:
         return LexicalResult(text=source, reason="parse_failed")
 
-    effective_min = max(0.08, min_wsd * 0.7) if aggressive else min_wsd
-    verb_gain_floor = 0.40 if aggressive else 0.45
-    adj_adv_gain_floor = 0.15 if aggressive else 0.0
+    if aggressive:
+        effective_min = max(0.08, min_wsd * 0.7)
+        verb_gain_floor = 0.40
+        adj_adv_gain_floor = 0.15
+    elif polish:
+        # Polish mode should feel stronger on any sentence, not only because the
+        # sentence earned a larger budget. Keep the same safety rails, but allow
+        # slightly weaker-yet-still-supported everyday substitutions.
+        effective_min = max(0.10, min_wsd * 0.85)
+        verb_gain_floor = 0.30
+        adj_adv_gain_floor = 0.10
+    else:
+        effective_min = min_wsd
+        verb_gain_floor = 0.45
+        adj_adv_gain_floor = 0.0
     stop_words = set(nlp.Defaults.stop_words)
     proposals: list[tuple[float, int, float, int, int, str, Any]] = []
     for token in doc:
@@ -623,12 +635,14 @@ def refine_sentence(
                 )
                 if (
                     best_score < noun_min
-                    or simplicity_gain < 0.35
-                    or simplicity_gain > 1.25
+                    or simplicity_gain < (0.25 if polish and not aggressive else 0.35)
+                    or simplicity_gain > (1.45 if polish and not aggressive else 1.25)
                 ):
                     continue
                 cand_freq = zipf_frequency(replacement.lower(), "en")
-                if cand_freq >= 5.5 and simplicity_gain >= 0.5:
+                if cand_freq >= 5.5 and simplicity_gain >= (
+                    0.7 if polish and not aggressive else 0.5
+                ):
                     continue
                 if best_score < effective_min:
                     best_score = effective_min
@@ -656,6 +670,11 @@ def refine_sentence(
             elif len(synsets) == 1:
                 chosen = best_synset
                 best_score = effective_min
+            elif polish and not aggressive and best_score >= effective_min * 0.75:
+                trial = _candidate_for_synset(best_synset, token, doc)
+                if trial is not None and trial[1] >= adj_adv_gain_floor:
+                    chosen = best_synset
+                    best_score = max(best_score, effective_min)
             elif ENGINE_LEXICAL_PREFER_SIMPLER and best_score == 0.0:
                 trial = _candidate_for_synset(synsets[0], token, doc)
                 gain_need = 0.25 if aggressive else 0.45
@@ -728,6 +747,30 @@ def refine_sentence(
                 confidence=round(confidence, 4),
             )
         )
+    if polish and not aggressive and len(changes) < limit:
+        # Polish mode gets one more pass with the aggressive-safe lexical rules.
+        extra = refine_sentence(
+            output,
+            min_wsd=max(0.08, min_wsd * 0.8),
+            max_changes=limit - len(changes),
+            wordnet=resource,
+            aggressive=True,
+            polish=False,
+        )
+        if extra.changes and extra.text != output:
+            seen = {
+                (change.original.lower(), change.replacement.lower())
+                for change in changes
+            }
+            appended = False
+            for change in extra.changes:
+                pair = (change.original.lower(), change.replacement.lower())
+                if pair not in seen:
+                    changes.append(change)
+                    seen.add(pair)
+                    appended = True
+            if appended:
+                output = extra.text
     changes.sort(key=lambda change: change.token_index)
     confidence = min(change.confidence for change in changes)
     return LexicalResult(text=output, changes=changes, confidence=confidence)

app/engine/orchestrator.py

@@ -24,7 +24,11 @@ from app.engine.consistency import apply_consistency
 from app.engine.force import force_cleft_rewrite
 from app.engine.grammar import repair_sentence
 from app.engine.ingest import ingest_text
-from app.engine.lexical import ensure_wording_change, refine_sentence
+from app.engine.lexical import (
+    dynamic_lexical_budget,
+    ensure_wording_change,
+    refine_sentence,
+)
 from app.engine.models import (
     DocumentBlock,
     EngineResult,
@@ -113,6 +117,64 @@ def _apply_lexical_refinement(
     return record
 
 
+def _apply_extra_polish(
+    record: SentenceRecord,
+    *,
+    enabled: bool,
+    min_wsd: float,
+    max_changes: int | None,
+    safety_min: float,
+    min_confidence: float,
+    use_minilm: bool,
+) -> SentenceRecord:
+    """Polish-only extra pass so the UI toggle has a real end-to-end effect."""
+    if not enabled or record.sentence_type not in _REWRITEABLE_TYPES:
+        return record
+    before = record.rewritten
+    target_budget = (
+        max(1, min(int(max_changes), 15))
+        if max_changes is not None
+        else dynamic_lexical_budget(before, polish=True)
+    )
+    remaining = max(0, target_budget - len(record.lexical_changes))
+    if remaining <= 0:
+        return record
+    refined = refine_sentence(
+        before,
+        min_wsd=min(min_wsd, 0.10),
+        max_changes=remaining,
+        aggressive=True,
+    )
+    if not refined.changes or refined.text == before:
+        return record
+    safety = check_safety(
+        before,
+        refined.text,
+        min_meaning=safety_min,
+        min_confidence=min_confidence,
+        use_minilm=use_minilm,
+        protected_entities=None,
+        protected_auxiliaries=None,
+        structural_validation=False,
+    )
+    if not safety.ok:
+        return record
+
+    prior_lex = list(record.lexical_changes)
+    record.rewritten = refined.text
+    record.lexical_changes = prior_lex + refined.changes
+    record.status = "rewritten"
+    if not record.template_id:
+        record.template_id = "lexical_refine"
+    elif "+lexical" not in record.template_id:
+        record.template_id = f"{record.template_id}+lexical"
+    record.confidence = min(
+        safety.confidence,
+        record.confidence if record.confidence > 0 else max(min_confidence, 0.55),
+    )
+    return record
+
+
 def _unchanged(record: SentenceRecord) -> bool:
     return (
         record.rewritten.strip().lower().rstrip(".!?")
@@ -311,15 +373,26 @@ def _finalize_sentence(
         use_minilm=use_minilm,
     )
     record = _apply_forced_rewrite(record, enabled=force_enabled)
-    record = _apply_lexical_refinement(
+    should_run_second_lexical = lexical_polish or not record.lexical_changes
+    if should_run_second_lexical:
+        record = _apply_lexical_refinement(
+            record,
+            enabled=lexical_enabled,
+            min_wsd=lexical_min_wsd,
+            max_changes=lexical_max_changes,
+            safety_min=safety_min,
+            min_confidence=min_confidence,
+            use_minilm=use_minilm,
+            polish=lexical_polish,
+        )
+    record = _apply_extra_polish(
         record,
-        enabled=lexical_enabled,
+        enabled=lexical_enabled and lexical_polish,
         min_wsd=lexical_min_wsd,
         max_changes=lexical_max_changes,
         safety_min=safety_min,
         min_confidence=min_confidence,
         use_minilm=use_minilm,
-        polish=lexical_polish,
     )
     return _ensure_wording_changed(
         record,