Pygmales commited on
Commit
6dc2f9f
·
verified ·
1 Parent(s): 05a5e3f

Sync from GitHub e41c1493d83f777145673b3f2273c84efb787b11

Browse files
src/rag/input_handler.py CHANGED
@@ -212,7 +212,11 @@ class InputHandler:
212
  and any(pattern in joined_words for pattern in InputHandler.KEYBOARD_MASH_PATTERNS)
213
  ):
214
  return True
215
- if InputHandler._looks_like_nonsense_word(joined_words):
 
 
 
 
216
  return True
217
  if any(InputHandler._looks_like_nonsense_word(word) for word in words):
218
  return True
@@ -223,15 +227,18 @@ class InputHandler:
223
  ):
224
  return True
225
 
226
- vowel_count = sum(1 for char in joined_words if char in InputHandler.VOWELS)
227
- if len(joined_words) >= 8 and vowel_count / len(joined_words) <= 0.18:
228
- return True
 
 
 
229
 
 
 
 
230
  for word in words:
231
- if len(word) >= 8 and (
232
- InputHandler._has_long_consonant_run(word)
233
- or not any(char in InputHandler.VOWELS for char in word)
234
- ):
235
  return True
236
 
237
  return False
 
212
  and any(pattern in joined_words for pattern in InputHandler.KEYBOARD_MASH_PATTERNS)
213
  ):
214
  return True
215
+ # Nonsense scoring on a concatenation only holds for compact input
216
+ # (e.g. a mash split by stray punctuation). On a real multi-word
217
+ # sentence it false-positives because word boundaries fabricate
218
+ # suspicious sequences and consonant runs.
219
+ if len(words) <= 2 and InputHandler._looks_like_nonsense_word(joined_words):
220
  return True
221
  if any(InputHandler._looks_like_nonsense_word(word) for word in words):
222
  return True
 
227
  ):
228
  return True
229
 
230
+ # Vowel-ratio is only a reliable gibberish signal for a single token;
231
+ # across a whole sentence it is meaningless.
232
+ if len(words) == 1:
233
+ vowel_count = sum(1 for char in joined_words if char in InputHandler.VOWELS)
234
+ if len(joined_words) >= 8 and vowel_count / len(joined_words) <= 0.18:
235
+ return True
236
 
237
+ # A long consonant run alone is a poor signal: German compounds such as
238
+ # "deutschsprachige" legitimately contain them. Only a complete absence
239
+ # of vowels in a long token is treated as gibberish here.
240
  for word in words:
241
+ if len(word) >= 8 and not any(char in InputHandler.VOWELS for char in word):
 
 
 
242
  return True
243
 
244
  return False
tests/test_input_handler_gibberish.py CHANGED
@@ -42,6 +42,12 @@ def test_process_input_rejects_probable_gibberish(message):
42
  "Zurich 8000",
43
  "leadership 2026",
44
  "Was sind die nächsten Schritte im Bewerbungsprozess?",
 
 
 
 
 
 
45
  "Ich interessiere mich für ein MBA mit Schwerpunkt nachhaltige Unternehmensführung.\nberufsbegleitend.",
46
  "Ich habe gerade meinen Bachelor abgeschlossen und 2 Jahre Berufserfahrung. Kann ich mich für den Executive MBA bewerben?",
47
  ],
 
42
  "Zurich 8000",
43
  "leadership 2026",
44
  "Was sind die nächsten Schritte im Bewerbungsprozess?",
45
+ # Regression: real queries the gibberish heuristic wrongly rejected.
46
+ # German compound words contain long consonant runs ("deutschsprachige");
47
+ # long English sentences false-positived once their words were concatenated.
48
+ "Wie lange dauert der deutschsprachige EMBA HSG?",
49
+ "Give me a short overview of all three executive MBA programmes.",
50
+ "Welche Weiterbildungsmöglichkeiten bietet die Universität?",
51
  "Ich interessiere mich für ein MBA mit Schwerpunkt nachhaltige Unternehmensführung.\nberufsbegleitend.",
52
  "Ich habe gerade meinen Bachelor abgeschlossen und 2 Jahre Berufserfahrung. Kann ich mich für den Executive MBA bewerben?",
53
  ],