Spaces:
Running
Running
Sync from GitHub e41c1493d83f777145673b3f2273c84efb787b11
Browse files
src/rag/input_handler.py
CHANGED
|
@@ -212,7 +212,11 @@ class InputHandler:
|
|
| 212 |
and any(pattern in joined_words for pattern in InputHandler.KEYBOARD_MASH_PATTERNS)
|
| 213 |
):
|
| 214 |
return True
|
| 215 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 216 |
return True
|
| 217 |
if any(InputHandler._looks_like_nonsense_word(word) for word in words):
|
| 218 |
return True
|
|
@@ -223,15 +227,18 @@ class InputHandler:
|
|
| 223 |
):
|
| 224 |
return True
|
| 225 |
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
|
|
|
|
|
|
|
|
|
| 229 |
|
|
|
|
|
|
|
|
|
|
| 230 |
for word in words:
|
| 231 |
-
if len(word) >= 8 and (
|
| 232 |
-
InputHandler._has_long_consonant_run(word)
|
| 233 |
-
or not any(char in InputHandler.VOWELS for char in word)
|
| 234 |
-
):
|
| 235 |
return True
|
| 236 |
|
| 237 |
return False
|
|
|
|
| 212 |
and any(pattern in joined_words for pattern in InputHandler.KEYBOARD_MASH_PATTERNS)
|
| 213 |
):
|
| 214 |
return True
|
| 215 |
+
# Nonsense scoring on a concatenation only holds for compact input
|
| 216 |
+
# (e.g. a mash split by stray punctuation). On a real multi-word
|
| 217 |
+
# sentence it false-positives because word boundaries fabricate
|
| 218 |
+
# suspicious sequences and consonant runs.
|
| 219 |
+
if len(words) <= 2 and InputHandler._looks_like_nonsense_word(joined_words):
|
| 220 |
return True
|
| 221 |
if any(InputHandler._looks_like_nonsense_word(word) for word in words):
|
| 222 |
return True
|
|
|
|
| 227 |
):
|
| 228 |
return True
|
| 229 |
|
| 230 |
+
# Vowel-ratio is only a reliable gibberish signal for a single token;
|
| 231 |
+
# across a whole sentence it is meaningless.
|
| 232 |
+
if len(words) == 1:
|
| 233 |
+
vowel_count = sum(1 for char in joined_words if char in InputHandler.VOWELS)
|
| 234 |
+
if len(joined_words) >= 8 and vowel_count / len(joined_words) <= 0.18:
|
| 235 |
+
return True
|
| 236 |
|
| 237 |
+
# A long consonant run alone is a poor signal: German compounds such as
|
| 238 |
+
# "deutschsprachige" legitimately contain them. Only a complete absence
|
| 239 |
+
# of vowels in a long token is treated as gibberish here.
|
| 240 |
for word in words:
|
| 241 |
+
if len(word) >= 8 and not any(char in InputHandler.VOWELS for char in word):
|
|
|
|
|
|
|
|
|
|
| 242 |
return True
|
| 243 |
|
| 244 |
return False
|
tests/test_input_handler_gibberish.py
CHANGED
|
@@ -42,6 +42,12 @@ def test_process_input_rejects_probable_gibberish(message):
|
|
| 42 |
"Zurich 8000",
|
| 43 |
"leadership 2026",
|
| 44 |
"Was sind die nächsten Schritte im Bewerbungsprozess?",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
"Ich interessiere mich für ein MBA mit Schwerpunkt nachhaltige Unternehmensführung.\nberufsbegleitend.",
|
| 46 |
"Ich habe gerade meinen Bachelor abgeschlossen und 2 Jahre Berufserfahrung. Kann ich mich für den Executive MBA bewerben?",
|
| 47 |
],
|
|
|
|
| 42 |
"Zurich 8000",
|
| 43 |
"leadership 2026",
|
| 44 |
"Was sind die nächsten Schritte im Bewerbungsprozess?",
|
| 45 |
+
# Regression: real queries the gibberish heuristic wrongly rejected.
|
| 46 |
+
# German compound words contain long consonant runs ("deutschsprachige");
|
| 47 |
+
# long English sentences false-positived once their words were concatenated.
|
| 48 |
+
"Wie lange dauert der deutschsprachige EMBA HSG?",
|
| 49 |
+
"Give me a short overview of all three executive MBA programmes.",
|
| 50 |
+
"Welche Weiterbildungsmöglichkeiten bietet die Universität?",
|
| 51 |
"Ich interessiere mich für ein MBA mit Schwerpunkt nachhaltige Unternehmensführung.\nberufsbegleitend.",
|
| 52 |
"Ich habe gerade meinen Bachelor abgeschlossen und 2 Jahre Berufserfahrung. Kann ich mich für den Executive MBA bewerben?",
|
| 53 |
],
|