Spaces:
Running on Zero
Running on Zero
feat(neural-search): add dynamic cross-encoder switch and multi-reranker selector for SuperAdmin
e690935 | """ | |
| Live Accuracy and Usage Benchmark for OpenRouter BGE-M3 + Cohere Rerank | |
| Sahih Islam Production Verification | |
| """ | |
| import sys | |
| import os | |
| import asyncio | |
| import time | |
| import numpy as np | |
| # Ensure UTF-8 output on Windows | |
| if sys.platform == "win32": | |
| sys.stdout.reconfigure(encoding="utf-8") | |
| backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| if backend_dir not in sys.path: | |
| sys.path.insert(0, backend_dir) | |
| from dotenv import load_dotenv | |
| load_dotenv(os.path.join(backend_dir, ".env")) | |
| from app.database import hadith_collection | |
| from app.inference_client import get_embedding, get_rerank_scores, DEFAULT_EMBEDDING_MODEL, DEFAULT_RERANKER_MODEL | |
| # 5 Core Islamic Test Queries with Verified Golden Targets | |
| BENCHMARK_QUERIES = [ | |
| { | |
| "id": 1, | |
| "query": "actions are judged by intentions", | |
| "topic": "Intentions (Niyyah)", | |
| "expected_keywords": ["intention", "niyyah", "emigrate", "reward", "action"], | |
| "famous_ref": "Sahih Bukhari Hadith 1 / Muslim 1907" | |
| }, | |
| { | |
| "id": 2, | |
| "query": "Islam is built upon five pillars", | |
| "topic": "Pillars of Islam (Arkan al-Islam)", | |
| "expected_keywords": ["pillar", "five", "prayer", "zakat", "fast", "hajj", "allah"], | |
| "famous_ref": "Sahih Bukhari Hadith 8 / Muslim 16" | |
| }, | |
| { | |
| "id": 3, | |
| "query": "smiling at your brother is charity", | |
| "topic": "Charity in Good Deeds", | |
| "expected_keywords": ["smile", "charity", "brother", "sadaqah", "good"], | |
| "famous_ref": "Jami at-Tirmidhi 1956" | |
| }, | |
| { | |
| "id": 4, | |
| "query": "the strong person is not the good wrestler but the one who controls anger", | |
| "topic": "Controlling Anger", | |
| "expected_keywords": ["wrestler", "strong", "anger", "angry", "controls"], | |
| "famous_ref": "Sahih Bukhari 6114 / Muslim 2609" | |
| }, | |
| { | |
| "id": 5, | |
| "query": "who is most deserving of my good companionship mother", | |
| "topic": "Rights of Parents (Mother)", | |
| "expected_keywords": ["mother", "father", "companionship", "treatment", "good"], | |
| "famous_ref": "Sahih Bukhari 5971 / Muslim 2548" | |
| } | |
| ] | |
| async def run_hadith_search(query: str, top_k: int = 3): | |
| t0 = time.time() | |
| # STEP 1: Bi-Encoder Dense Vectorization via OpenRouter BGE-M3 | |
| query_vec = (await get_embedding(query)).tolist() | |
| t_embed = time.time() - t0 | |
| # STEP 2: MongoDB Atlas Vector Search | |
| t1 = time.time() | |
| stage = { | |
| "$vectorSearch": { | |
| "index": "vector_index", | |
| "path": "embedding_1024", | |
| "queryVector": query_vec, | |
| "numCandidates": 200, | |
| "limit": 20 | |
| } | |
| } | |
| raw_candidates = await hadith_collection.aggregate([ | |
| stage, | |
| {"$addFields": {"_vscore": {"$meta": "vectorSearchScore"}}} | |
| ]).to_list(length=20) | |
| t_atlas = time.time() - t1 | |
| if not raw_candidates: | |
| return [], t_embed, t_atlas, 0.0 | |
| # STEP 3: Cross-Encoder Precision Reranking via OpenRouter Cohere Rerank v3.5 | |
| t2 = time.time() | |
| pairs = [[query, str(d.get("text_english", ""))[:1000]] for d in raw_candidates] | |
| scores = await get_rerank_scores(pairs) | |
| t_rerank = time.time() - t2 | |
| ranked_indices = np.argsort(scores)[::-1] | |
| final_results = [] | |
| for idx in ranked_indices[:top_k]: | |
| doc = raw_candidates[idx] | |
| doc["_rerank_score"] = float(scores[idx]) | |
| final_results.append(doc) | |
| return final_results, t_embed, t_atlas, t_rerank | |
| async def main(): | |
| print("=" * 80) | |
| print("SAHIH ISLAM - LIVE SEARCH ACCURACY BENCHMARK (OpenRouter Cloud Inference)") | |
| print(f"Bi-Encoder Model : {DEFAULT_EMBEDDING_MODEL}") | |
| print(f"Cross-Encoder : {DEFAULT_RERANKER_MODEL}") | |
| print(f"Total Test Runs : {len(BENCHMARK_QUERIES)} Searches") | |
| print("=" * 80) | |
| total_accuracy_score = 0 | |
| total_searches = len(BENCHMARK_QUERIES) | |
| for item in BENCHMARK_QUERIES: | |
| q_id = item["id"] | |
| query = item["query"] | |
| print(f"\n[SEARCH #{q_id}/{total_searches}] Query: \"{query}\"") | |
| print(f" Topic: {item['topic']} | Expected: {item['famous_ref']}") | |
| results, t_emb, t_atl, t_rer = await run_hadith_search(query, top_k=3) | |
| print(f" Timing: Embed={t_emb:.3f}s | Atlas={t_atl:.3f}s | Rerank={t_rer:.3f}s | Total={t_emb+t_atl+t_rer:.3f}s") | |
| if not results: | |
| print(" ❌ ZERO RESULTS RETURNED!") | |
| continue | |
| top_hit = results[0] | |
| ref = top_hit.get("reference") or f"{top_hit.get('collection')} #{top_hit.get('hadith_no')}" | |
| narrator = top_hit.get("narrator", "N/A") | |
| text_en = top_hit.get("text_english", "")[:280] | |
| vscore = top_hit.get("_vscore", 0.0) | |
| rscore = top_hit.get("_rerank_score", 0.0) | |
| # Check keyword presence in top 3 hits | |
| found_keywords = [kw for kw in item["expected_keywords"] if kw in text_en.lower()] | |
| accuracy_match = len(found_keywords) >= 2 | |
| if accuracy_match: | |
| total_accuracy_score += 1 | |
| status_icon = "[EXACT MATCH]" | |
| else: | |
| status_icon = "[PARTIAL MATCH]" | |
| print(f" {status_icon} -> Top Rank #1:") | |
| print(f" Reference : {ref}") | |
| print(f" Narrator : {narrator}") | |
| print(f" Vector Sim : {vscore:.4f} | Rerank Score: {rscore:.4f}") | |
| print(f" Text Preview: \"{text_en}...\"") | |
| print("\n" + "=" * 80) | |
| print("BENCHMARK SUMMARY") | |
| print("=" * 80) | |
| print(f"Total Searches Executed : {total_searches}") | |
| print(f"Total OpenRouter BGE-M3 Calls : {total_searches}") | |
| print(f"Total OpenRouter Rerank Calls : {total_searches}") | |
| print(f"Search Accuracy Rate : {total_accuracy_score}/{total_searches} ({(total_accuracy_score/total_searches)*100:.1f}%)") | |
| print("=" * 80) | |
| if __name__ == "__main__": | |
| asyncio.run(main()) | |