Commit ·
ca212fd
1
Parent(s): cfb4439
Phase 14c: Fix FIX-31 whitespace bug + fix misleading log truncation
Browse filesFIX-31: .rstrip() before checking trailing punct regex — model may
output trailing whitespace after hallucinated dots, breaking the
dollar-sign anchor.
Log fix: Grammar and Punctuation service logs were hardcoding '...'
after [:80] truncation even for short texts. This made it appear
that the models were adding ellipsis when they weren't. Now only
appends '...' when text is actually >80 chars.
src/app.py
CHANGED
|
@@ -1531,11 +1531,15 @@ def analyze_text():
|
|
| 1531 |
# ── Phase 14 (FIX-31): Strip hallucinated trailing punctuation ──
|
| 1532 |
# The AraSpell model sometimes hallucinates trailing '...' or '.'
|
| 1533 |
# that weren't in the input. Strip them to prevent dot accumulation.
|
|
|
|
|
|
|
| 1534 |
import re as _re_strip
|
| 1535 |
-
|
| 1536 |
-
|
|
|
|
|
|
|
| 1537 |
if _output_trailing and not _input_trailing:
|
| 1538 |
-
raw_corrected =
|
| 1539 |
logger.info(
|
| 1540 |
f"[SPELLING] Stripped hallucinated trailing punct: "
|
| 1541 |
f"'{_output_trailing.group()}'"
|
|
@@ -1543,7 +1547,7 @@ def analyze_text():
|
|
| 1543 |
elif _output_trailing and _input_trailing:
|
| 1544 |
# If input had some trailing punct, preserve only what was there
|
| 1545 |
if len(_output_trailing.group()) > len(_input_trailing.group()):
|
| 1546 |
-
raw_corrected =
|
| 1547 |
logger.info(
|
| 1548 |
f"[SPELLING] Trimmed extra trailing punct: "
|
| 1549 |
f"'{_output_trailing.group()}' → '{_input_trailing.group()}'"
|
|
|
|
| 1531 |
# ── Phase 14 (FIX-31): Strip hallucinated trailing punctuation ──
|
| 1532 |
# The AraSpell model sometimes hallucinates trailing '...' or '.'
|
| 1533 |
# that weren't in the input. Strip them to prevent dot accumulation.
|
| 1534 |
+
# NOTE: Must .rstrip() first — model may add trailing whitespace
|
| 1535 |
+
# after dots, breaking the $ anchor.
|
| 1536 |
import re as _re_strip
|
| 1537 |
+
_rc_stripped = raw_corrected.rstrip()
|
| 1538 |
+
_ct_stripped = current_text.rstrip()
|
| 1539 |
+
_input_trailing = _re_strip.search(r'[\.،؛؟!]+$', _ct_stripped)
|
| 1540 |
+
_output_trailing = _re_strip.search(r'[\.،؛؟!]+$', _rc_stripped)
|
| 1541 |
if _output_trailing and not _input_trailing:
|
| 1542 |
+
raw_corrected = _rc_stripped[:_output_trailing.start()]
|
| 1543 |
logger.info(
|
| 1544 |
f"[SPELLING] Stripped hallucinated trailing punct: "
|
| 1545 |
f"'{_output_trailing.group()}'"
|
|
|
|
| 1547 |
elif _output_trailing and _input_trailing:
|
| 1548 |
# If input had some trailing punct, preserve only what was there
|
| 1549 |
if len(_output_trailing.group()) > len(_input_trailing.group()):
|
| 1550 |
+
raw_corrected = _rc_stripped[:_output_trailing.start()] + _input_trailing.group()
|
| 1551 |
logger.info(
|
| 1552 |
f"[SPELLING] Trimmed extra trailing punct: "
|
| 1553 |
f"'{_output_trailing.group()}' → '{_input_trailing.group()}'"
|
src/nlp/grammar/grammar_service.py
CHANGED
|
@@ -98,7 +98,9 @@ class GrammarChecker:
|
|
| 98 |
text=text,
|
| 99 |
api_name="/correct_grammar"
|
| 100 |
)
|
| 101 |
-
|
|
|
|
|
|
|
| 102 |
|
| 103 |
if not model_output or not model_output.strip():
|
| 104 |
logger.warning("Grammar model returned empty output, returning original")
|
|
|
|
| 98 |
text=text,
|
| 99 |
api_name="/correct_grammar"
|
| 100 |
)
|
| 101 |
+
_mo_display = model_output[:80] + ('...' if len(model_output) > 80 else '')
|
| 102 |
+
_ti_display = text[:80] + ('...' if len(text) > 80 else '')
|
| 103 |
+
logger.info(f"Grammar model output: '{_mo_display}' (input: '{_ti_display}')")
|
| 104 |
|
| 105 |
if not model_output or not model_output.strip():
|
| 106 |
logger.warning("Grammar model returned empty output, returning original")
|
src/nlp/punctuation/punctuation_service.py
CHANGED
|
@@ -214,7 +214,9 @@ class PunctuationChecker:
|
|
| 214 |
processed_paragraphs.append(cleaned)
|
| 215 |
|
| 216 |
result = "\n\n".join(processed_paragraphs)
|
| 217 |
-
|
|
|
|
|
|
|
| 218 |
return result
|
| 219 |
|
| 220 |
except Exception as e:
|
|
|
|
| 214 |
processed_paragraphs.append(cleaned)
|
| 215 |
|
| 216 |
result = "\n\n".join(processed_paragraphs)
|
| 217 |
+
_r_display = result[:80] + ('...' if len(result) > 80 else '')
|
| 218 |
+
_t_display = text[:80] + ('...' if len(text) > 80 else '')
|
| 219 |
+
logger.info(f"Punctuation output: '{_r_display}' (input: '{_t_display}')")
|
| 220 |
return result
|
| 221 |
|
| 222 |
except Exception as e:
|