Update app.py

app.py

@@ -4,6 +4,7 @@ HuggingFace Spaces / Gradio
 """
 
 import re
+from transformers import BertTokenizer
 import gradio as gr
 from datasets import load_dataset
 from rank_bm25 import BM25Okapi
@@ -32,30 +33,17 @@ print(f"Corpus hazir: {len(doc_ids):,} dokuman")
 # 2. TOKENİZERS
 # ══════════════════════════════════════════════════════════════════════
 
+# Whitespace tokenizer: Python split() bazlı
 def whitespace_tokenize(text):
-    return re.findall(r'\b[a-z]+\b', text.lower())
+    return text.lower().split()
 
-
-_SUFFIXES = [
-    'ization', 'isation', 'ation', 'tion', 'sion', 'ment', 'ness',
-    'ity', 'ical', 'ous', 'ful', 'less', 'ize', 'ise',
-    'ing', 'al', 'er', 'est', 'ly', 'ed',
-]
+# BERT tokenizer: HuggingFace bert-base-uncased
+print("BERT tokenizer yukleniyor...")
+bert_tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
+print("BERT tokenizer hazir.")
 
 def bert_tokenize(text):
-    tokens = []
-    for word in re.findall(r"[a-z]+(?:-[a-z]+)*", text.lower()):
-        for part in word.split('-'):
-            matched = False
-            for suf in sorted(_SUFFIXES, key=len, reverse=True):
-                if len(part) > len(suf) + 2 and part.endswith(suf):
-                    tokens.append(part[:-len(suf)])
-                    tokens.append('##' + suf)
-                    matched = True
-                    break
-            if not matched:
-                tokens.append(part)
-    return tokens
+    return bert_tokenizer.tokenize(text)
 
 
 # ══════════════════════════════════════════════════════════════════════