fix: Word-level grammar suggestions + robust grammar model validation

Grammar service:
- Add word count validation (reject if ratio < 0.5 or > 2.0)
- Add repetition detection (reject if 2+ consecutive repeated words)
- Add length check (reject if output > 2x input)
- Add character overlap check (reject if < 50% chars preserved)

Grammar diff in app.py:
- Break large replace blocks into individual word suggestions
- When word counts match, process EACH word pair separately
- Skip large blocks (>3 words) to prevent hallucination suggestions
- Each grammar suggestion now points to a single word, not entire sentence

src/app.py

@@ -911,49 +911,91 @@ def analyze_text():
                         if tag != 'replace':
                             continue
 
-                        # Process each changed word individually
-                        g_segment = grammar_words[j1:j2]
-                        correction_text = " ".join(g_segment)
-
-                        # Map back to original text positions
-                        # Find the earliest and latest original word indices
-                        orig_indices = set()
-                        for k in range(i1, i2):
-                            if k < len(spelling_word_map):
-                                orig_indices.add(spelling_word_map[k])
-
-                        if not orig_indices:
-                            continue
-
-                        min_orig = min(orig_indices)
-                        max_orig = max(orig_indices)
-
-                        if min_orig >= len(orig_word_positions) or max_orig >= len(orig_word_positions):
-                            continue
-
-                        orig_start = orig_word_positions[min_orig][1]
-                        orig_end = orig_word_positions[max_orig][2]
-                        original_span = text[orig_start:orig_end]
-
-                        # Skip if the correction is the same as original
-                        if original_span == correction_text:
-                            continue
-
-                        # Skip if this overlaps with any spelling suggestion
-                        overlaps = any(
-                            not (orig_end <= sr_start or orig_start >= sr_end)
-                            for sr_start, sr_end in spelling_ranges
-                        )
-                        if overlaps:
-                            continue
-
-                        suggestions.append({
-                            'start': orig_start,
-                            'end': orig_end,
-                            'original': original_span,
-                            'correction': correction_text,
-                            'type': 'grammar'
-                        })
+                        o_count = i2 - i1
+                        c_count = j2 - j1
+
+                        # If word counts match, process EACH word individually
+                        if o_count == c_count:
+                            for offset in range(o_count):
+                                sw = spell_words[i1 + offset]
+                                gw = grammar_words[j1 + offset]
+                                if sw == gw:
+                                    continue  # no change at this position
+
+                                # Map to original text position
+                                spell_idx = i1 + offset
+                                if spell_idx >= len(spelling_word_map):
+                                    continue
+                                orig_idx = spelling_word_map[spell_idx]
+                                if orig_idx >= len(orig_word_positions):
+                                    continue
+
+                                orig_start = orig_word_positions[orig_idx][1]
+                                orig_end = orig_word_positions[orig_idx][2]
+                                original_word = text[orig_start:orig_end]
+
+                                if original_word == gw:
+                                    continue
+
+                                # Overlap check
+                                overlaps = any(
+                                    not (orig_end <= sr_s or orig_start >= sr_e)
+                                    for sr_s, sr_e in spelling_ranges
+                                )
+                                if overlaps:
+                                    continue
+
+                                suggestions.append({
+                                    'start': orig_start,
+                                    'end': orig_end,
+                                    'original': original_word,
+                                    'correction': gw,
+                                    'type': 'grammar'
+                                })
+                        else:
+                            # Word count differs — create one grouped suggestion
+                            # but only if it's a small block (max 3 words)
+                            if o_count > 3 or c_count > 3:
+                                continue  # skip large blocks (likely hallucination)
+
+                            g_segment = grammar_words[j1:j2]
+                            correction_text = " ".join(g_segment)
+
+                            orig_indices = set()
+                            for k in range(i1, i2):
+                                if k < len(spelling_word_map):
+                                    orig_indices.add(spelling_word_map[k])
+
+                            if not orig_indices:
+                                continue
+
+                            min_orig = min(orig_indices)
+                            max_orig = max(orig_indices)
+
+                            if min_orig >= len(orig_word_positions) or max_orig >= len(orig_word_positions):
+                                continue
+
+                            orig_start = orig_word_positions[min_orig][1]
+                            orig_end = orig_word_positions[max_orig][2]
+                            original_span = text[orig_start:orig_end]
+
+                            if original_span == correction_text:
+                                continue
+
+                            overlaps = any(
+                                not (orig_end <= sr_s or orig_start >= sr_e)
+                                for sr_s, sr_e in spelling_ranges
+                            )
+                            if overlaps:
+                                continue
+
+                            suggestions.append({
+                                'start': orig_start,
+                                'end': orig_end,
+                                'original': original_span,
+                                'correction': correction_text,
+                                'type': 'grammar'
+                            })
 
                     current_text = corrected_grammar
             except Exception as e:

src/nlp/grammar/grammar_service.py

@@ -102,16 +102,54 @@ def correct_grammar(text: str) -> str:
         # Decode output
         corrected = tokenizer.decode(outputs[0], skip_special_tokens=True)
 
-        # Safety: if model returns empty or much shorter, keep original
+        # === VALIDATION ===
+
+        # 1. Empty check
         if not corrected or not corrected.strip():
             return text
 
-        # If the correction is drastically different (>80% changed), keep original
+        corrected = corrected.strip()
+
+        # 2. Length check: too short = hallucination
         if len(corrected) < len(text) * 0.3:
             logger.warning(f"Grammar output too short ({len(corrected)} vs {len(text)}), keeping original")
             return text
 
-        return corrected.strip()
+        # 3. Length check: too long = repetition/hallucination
+        if len(corrected) > len(text) * 2.0:
+            logger.warning(f"Grammar output too long ({len(corrected)} vs {len(text)}), keeping original")
+            return text
+
+        # 4. Word count check: grammar shouldn't drastically change word count
+        orig_wc = len(text.split())
+        corr_wc = len(corrected.split())
+        if orig_wc > 0:
+            ratio = corr_wc / orig_wc
+            if ratio < 0.5 or ratio > 2.0:
+                logger.warning(f"Grammar word count mismatch ({corr_wc} vs {orig_wc}), keeping original")
+                return text
+
+        # 5. Repetition detection: reject if output has excessive repeated words
+        corr_words = corrected.split()
+        if len(corr_words) >= 4:
+            consecutive_repeats = 0
+            for i in range(1, len(corr_words)):
+                if corr_words[i] == corr_words[i-1]:
+                    consecutive_repeats += 1
+            if consecutive_repeats >= 2:
+                logger.warning(f"Grammar output has excessive repetition ({consecutive_repeats} repeats), keeping original")
+                return text
+
+        # 6. Character overlap check: grammar should preserve most of the content
+        orig_chars = set(text.replace(' ', ''))
+        corr_chars = set(corrected.replace(' ', ''))
+        if orig_chars:
+            overlap = len(orig_chars & corr_chars) / len(orig_chars)
+            if overlap < 0.5:
+                logger.warning(f"Grammar output low char overlap ({overlap:.2f}), keeping original")
+                return text
+
+        return corrected
 
     except Exception as e:
         logger.error(f"Grammar correction failed: {e}")