youssefreda9 commited on
Commit
ca212fd
·
1 Parent(s): cfb4439

Phase 14c: Fix FIX-31 whitespace bug + fix misleading log truncation

Browse files

FIX-31: .rstrip() before checking trailing punct regex — model may
output trailing whitespace after hallucinated dots, breaking the
dollar-sign anchor.

Log fix: Grammar and Punctuation service logs were hardcoding '...'
after [:80] truncation even for short texts. This made it appear
that the models were adding ellipsis when they weren't. Now only
appends '...' when text is actually >80 chars.

src/app.py CHANGED
@@ -1531,11 +1531,15 @@ def analyze_text():
1531
  # ── Phase 14 (FIX-31): Strip hallucinated trailing punctuation ──
1532
  # The AraSpell model sometimes hallucinates trailing '...' or '.'
1533
  # that weren't in the input. Strip them to prevent dot accumulation.
 
 
1534
  import re as _re_strip
1535
- _input_trailing = _re_strip.search(r'[\.،؛؟!]+$', current_text)
1536
- _output_trailing = _re_strip.search(r'[\.،؛؟!]+$', raw_corrected)
 
 
1537
  if _output_trailing and not _input_trailing:
1538
- raw_corrected = raw_corrected[:_output_trailing.start()]
1539
  logger.info(
1540
  f"[SPELLING] Stripped hallucinated trailing punct: "
1541
  f"'{_output_trailing.group()}'"
@@ -1543,7 +1547,7 @@ def analyze_text():
1543
  elif _output_trailing and _input_trailing:
1544
  # If input had some trailing punct, preserve only what was there
1545
  if len(_output_trailing.group()) > len(_input_trailing.group()):
1546
- raw_corrected = raw_corrected[:_output_trailing.start()] + _input_trailing.group()
1547
  logger.info(
1548
  f"[SPELLING] Trimmed extra trailing punct: "
1549
  f"'{_output_trailing.group()}' → '{_input_trailing.group()}'"
 
1531
  # ── Phase 14 (FIX-31): Strip hallucinated trailing punctuation ──
1532
  # The AraSpell model sometimes hallucinates trailing '...' or '.'
1533
  # that weren't in the input. Strip them to prevent dot accumulation.
1534
+ # NOTE: Must .rstrip() first — model may add trailing whitespace
1535
+ # after dots, breaking the $ anchor.
1536
  import re as _re_strip
1537
+ _rc_stripped = raw_corrected.rstrip()
1538
+ _ct_stripped = current_text.rstrip()
1539
+ _input_trailing = _re_strip.search(r'[\.،؛؟!]+$', _ct_stripped)
1540
+ _output_trailing = _re_strip.search(r'[\.،؛؟!]+$', _rc_stripped)
1541
  if _output_trailing and not _input_trailing:
1542
+ raw_corrected = _rc_stripped[:_output_trailing.start()]
1543
  logger.info(
1544
  f"[SPELLING] Stripped hallucinated trailing punct: "
1545
  f"'{_output_trailing.group()}'"
 
1547
  elif _output_trailing and _input_trailing:
1548
  # If input had some trailing punct, preserve only what was there
1549
  if len(_output_trailing.group()) > len(_input_trailing.group()):
1550
+ raw_corrected = _rc_stripped[:_output_trailing.start()] + _input_trailing.group()
1551
  logger.info(
1552
  f"[SPELLING] Trimmed extra trailing punct: "
1553
  f"'{_output_trailing.group()}' → '{_input_trailing.group()}'"
src/nlp/grammar/grammar_service.py CHANGED
@@ -98,7 +98,9 @@ class GrammarChecker:
98
  text=text,
99
  api_name="/correct_grammar"
100
  )
101
- logger.info(f"Grammar model output: '{model_output[:80]}...' (input: '{text[:80]}...')")
 
 
102
 
103
  if not model_output or not model_output.strip():
104
  logger.warning("Grammar model returned empty output, returning original")
 
98
  text=text,
99
  api_name="/correct_grammar"
100
  )
101
+ _mo_display = model_output[:80] + ('...' if len(model_output) > 80 else '')
102
+ _ti_display = text[:80] + ('...' if len(text) > 80 else '')
103
+ logger.info(f"Grammar model output: '{_mo_display}' (input: '{_ti_display}')")
104
 
105
  if not model_output or not model_output.strip():
106
  logger.warning("Grammar model returned empty output, returning original")
src/nlp/punctuation/punctuation_service.py CHANGED
@@ -214,7 +214,9 @@ class PunctuationChecker:
214
  processed_paragraphs.append(cleaned)
215
 
216
  result = "\n\n".join(processed_paragraphs)
217
- logger.info(f"Punctuation output: '{result[:80]}...' (input: '{text[:80]}...')")
 
 
218
  return result
219
 
220
  except Exception as e:
 
214
  processed_paragraphs.append(cleaned)
215
 
216
  result = "\n\n".join(processed_paragraphs)
217
+ _r_display = result[:80] + ('...' if len(result) > 80 else '')
218
+ _t_display = text[:80] + ('...' if len(text) > 80 else '')
219
+ logger.info(f"Punctuation output: '{_r_display}' (input: '{_t_display}')")
220
  return result
221
 
222
  except Exception as e: