Upload kb_reindex.py with huggingface_hub

kb_reindex.py

@@ -0,0 +1,17 @@
+"""Re-index existing passages with title boosting (title repeated to raise its
+BM25 weight) so the canonical article ranks higher for entity questions."""
+import json, time
+import bm25s, Stemmer
+
+KB = "/root/kb"
+stemmer = Stemmer.Stemmer("russian")
+P = [json.loads(l) for l in open(f"{KB}/passages.jsonl")]
+# repeat title 3x to boost title-term frequency
+corpus = [f"{p['title']}. {p['title']}. {p['title']}. {p['text']}" for p in P]
+t0 = time.time()
+ctok = bm25s.tokenize(corpus, stopwords=None, stemmer=stemmer, show_progress=False)
+retr = bm25s.BM25(k1=1.5, b=0.75)
+retr.index(ctok, show_progress=False)
+retr.save(f"{KB}/bm25_index")
+print(f"reindexed {len(P)} docs in {time.time()-t0:.0f}s", flush=True)
+print("REIDX_DONE", flush=True)