first_commit

app.py

@@ -0,0 +1,157 @@
+"""
+Tokenization Impact on Retrieval — TREC-COVID Demo
+HuggingFace Spaces / Gradio
+"""
+
+import re
+import gradio as gr
+from datasets import load_dataset
+from rank_bm25 import BM25Okapi
+
+# ══════════════════════════════════════════════════════════════════════
+# 1. CORPUS İNDİR & İNDEKS KUR (uygulama başlarken bir kere)
+# ══════════════════════════════════════════════════════════════════════
+
+print("TREC-COVID corpus indiriliyor...")
+corpus_ds = load_dataset("BeIR/trec-covid", "corpus", split="corpus")
+
+corpus_title = {}
+corpus_dict  = {}
+for doc in corpus_ds:
+    did   = str(doc["_id"])
+    title = doc["title"] if doc["title"] else doc["text"][:120]
+    corpus_title[did] = title
+    corpus_dict[did]  = title + " " + doc["text"]
+
+doc_ids   = list(corpus_dict.keys())
+doc_texts = [corpus_dict[did] for did in doc_ids]
+print(f"Corpus hazir: {len(doc_ids):,} dokuman")
+
+
+# ══════════════════════════════════════════════════════════════════════
+# 2. TOKENİZERS
+# ══════════════════════════════════════════════════════════════════════
+
+def whitespace_tokenize(text):
+    return re.findall(r'\b[a-z]+\b', text.lower())
+
+
+_SUFFIXES = [
+    'ization', 'isation', 'ation', 'tion', 'sion', 'ment', 'ness',
+    'ity', 'ical', 'ous', 'ful', 'less', 'ize', 'ise',
+    'ing', 'al', 'er', 'est', 'ly', 'ed',
+]
+
+def bert_tokenize(text):
+    tokens = []
+    for word in re.findall(r"[a-z]+(?:-[a-z]+)*", text.lower()):
+        for part in word.split('-'):
+            matched = False
+            for suf in sorted(_SUFFIXES, key=len, reverse=True):
+                if len(part) > len(suf) + 2 and part.endswith(suf):
+                    tokens.append(part[:-len(suf)])
+                    tokens.append('##' + suf)
+                    matched = True
+                    break
+            if not matched:
+                tokens.append(part)
+    return tokens
+
+
+# ══════════════════════════════════════════════════════════════════════
+# 3. BM25 İNDEKSLERİ
+# ══════════════════════════════════════════════════════════════════════
+
+print("BM25 indeksleri kuruluyor (birkaç dakika)...")
+bm25_ws   = BM25Okapi([whitespace_tokenize(t) for t in doc_texts])
+bm25_bert = BM25Okapi([bert_tokenize(t) for t in doc_texts])
+print("Hazir!")
+
+
+# ══════════════════════════════════════════════════════════════════════
+# 4. RETRIEVAL
+# ══════════════════════════════════════════════════════════════════════
+
+def retrieve(bm25, tokenize_fn, query, top_k=5):
+    tokens = tokenize_fn(query)
+    scores = bm25.get_scores(tokens)
+    ranked = sorted(enumerate(scores), key=lambda x: x[1], reverse=True)
+    return [(doc_ids[i], corpus_title[doc_ids[i]], round(s, 2)) for i, s in ranked[:top_k]]
+
+
+# ══════════════════════════════════════════════════════════════════════
+# 5. GRADIO ARAYÜZ
+# ══════════════════════════════════════════════════════════════════════
+
+def search(query):
+    if not query.strip():
+        return "Query boş olamaz.", "Query boş olamaz."
+
+    ws_tokens   = whitespace_tokenize(query)
+    bert_tokens = bert_tokenize(query)
+    ws_results   = retrieve(bm25_ws,   whitespace_tokenize, query)
+    bert_results = retrieve(bm25_bert, bert_tokenize,       query)
+
+    def format_tokens(tokens, style):
+        if style == "ws":
+            return " | ".join(f"`{t}`" for t in tokens)
+        else:
+            parts = []
+            for t in tokens:
+                if t.startswith("##"):
+                    parts.append(f"**`{t}`**")
+                else:
+                    parts.append(f"`{t}`")
+            return " | ".join(parts)
+
+    def format_results(results):
+        lines = []
+        for i, (did, title, score) in enumerate(results, 1):
+            lines.append(f"**{i}.** {title}  \n`score: {score}`")
+        return "\n\n---\n\n".join(lines)
+
+    ws_out = f"### ⬜ Whitespace Tokens\n{format_tokens(ws_tokens, 'ws')}\n\n---\n\n"
+    ws_out += f"### Top-5 Sonuçlar\n\n{format_results(ws_results)}"
+
+    bert_out = f"### 🔷 BERT-style Tokens\n{format_tokens(bert_tokens, 'bert')}\n\n---\n\n"
+    bert_out += f"### Top-5 Sonuçlar\n\n{format_results(bert_results)}"
+
+    return ws_out, bert_out
+
+
+examples = [
+    "what is the origin of COVID-19",
+    "how does coronavirus spread among people",
+    "COVID-19 symptoms fever cough loss of smell",
+    "remdesivir antiviral treatment efficacy",
+    "vaccine mRNA clinical trial efficacy",
+    "coronavirus incubation period transmission",
+    "comorbidities risk factors severe COVID",
+]
+
+with gr.Blocks(theme=gr.themes.Soft(), title="Tokenization Impact on Retrieval") as demo:
+    gr.Markdown("""
+    # 🔍 Tokenization Impact on Retrieval Quality
+    **TREC-COVID · BM25 · Whitespace vs BERT-style Tokenization**
+
+    Assignment 15 — Information Retrieval
+    """)
+
+    with gr.Row():
+        query_input = gr.Textbox(
+            placeholder="e.g. how does coronavirus spread among people",
+            label="Query",
+            scale=5,
+        )
+        search_btn = gr.Button("Search 🔍", variant="primary", scale=1)
+
+    gr.Examples(examples=examples, inputs=query_input, label="Example Queries")
+
+    with gr.Row():
+        ws_output   = gr.Markdown(label="⬜ Whitespace BM25")
+        bert_output = gr.Markdown(label="🔷 BERT-style BM25")
+
+    search_btn.click(fn=search, inputs=query_input, outputs=[ws_output, bert_output])
+    query_input.submit(fn=search, inputs=query_input, outputs=[ws_output, bert_output])
+
+demo.launch()