Spaces:
Running on Zero
Running on Zero
File size: 5,907 Bytes
e690935 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 | """
Live Accuracy and Usage Benchmark for OpenRouter BGE-M3 + Cohere Rerank
Sahih Islam Production Verification
"""
import sys
import os
import asyncio
import time
import numpy as np
# Ensure UTF-8 output on Windows
if sys.platform == "win32":
sys.stdout.reconfigure(encoding="utf-8")
backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if backend_dir not in sys.path:
sys.path.insert(0, backend_dir)
from dotenv import load_dotenv
load_dotenv(os.path.join(backend_dir, ".env"))
from app.database import hadith_collection
from app.inference_client import get_embedding, get_rerank_scores, DEFAULT_EMBEDDING_MODEL, DEFAULT_RERANKER_MODEL
# 5 Core Islamic Test Queries with Verified Golden Targets
BENCHMARK_QUERIES = [
{
"id": 1,
"query": "actions are judged by intentions",
"topic": "Intentions (Niyyah)",
"expected_keywords": ["intention", "niyyah", "emigrate", "reward", "action"],
"famous_ref": "Sahih Bukhari Hadith 1 / Muslim 1907"
},
{
"id": 2,
"query": "Islam is built upon five pillars",
"topic": "Pillars of Islam (Arkan al-Islam)",
"expected_keywords": ["pillar", "five", "prayer", "zakat", "fast", "hajj", "allah"],
"famous_ref": "Sahih Bukhari Hadith 8 / Muslim 16"
},
{
"id": 3,
"query": "smiling at your brother is charity",
"topic": "Charity in Good Deeds",
"expected_keywords": ["smile", "charity", "brother", "sadaqah", "good"],
"famous_ref": "Jami at-Tirmidhi 1956"
},
{
"id": 4,
"query": "the strong person is not the good wrestler but the one who controls anger",
"topic": "Controlling Anger",
"expected_keywords": ["wrestler", "strong", "anger", "angry", "controls"],
"famous_ref": "Sahih Bukhari 6114 / Muslim 2609"
},
{
"id": 5,
"query": "who is most deserving of my good companionship mother",
"topic": "Rights of Parents (Mother)",
"expected_keywords": ["mother", "father", "companionship", "treatment", "good"],
"famous_ref": "Sahih Bukhari 5971 / Muslim 2548"
}
]
async def run_hadith_search(query: str, top_k: int = 3):
t0 = time.time()
# STEP 1: Bi-Encoder Dense Vectorization via OpenRouter BGE-M3
query_vec = (await get_embedding(query)).tolist()
t_embed = time.time() - t0
# STEP 2: MongoDB Atlas Vector Search
t1 = time.time()
stage = {
"$vectorSearch": {
"index": "vector_index",
"path": "embedding_1024",
"queryVector": query_vec,
"numCandidates": 200,
"limit": 20
}
}
raw_candidates = await hadith_collection.aggregate([
stage,
{"$addFields": {"_vscore": {"$meta": "vectorSearchScore"}}}
]).to_list(length=20)
t_atlas = time.time() - t1
if not raw_candidates:
return [], t_embed, t_atlas, 0.0
# STEP 3: Cross-Encoder Precision Reranking via OpenRouter Cohere Rerank v3.5
t2 = time.time()
pairs = [[query, str(d.get("text_english", ""))[:1000]] for d in raw_candidates]
scores = await get_rerank_scores(pairs)
t_rerank = time.time() - t2
ranked_indices = np.argsort(scores)[::-1]
final_results = []
for idx in ranked_indices[:top_k]:
doc = raw_candidates[idx]
doc["_rerank_score"] = float(scores[idx])
final_results.append(doc)
return final_results, t_embed, t_atlas, t_rerank
async def main():
print("=" * 80)
print("SAHIH ISLAM - LIVE SEARCH ACCURACY BENCHMARK (OpenRouter Cloud Inference)")
print(f"Bi-Encoder Model : {DEFAULT_EMBEDDING_MODEL}")
print(f"Cross-Encoder : {DEFAULT_RERANKER_MODEL}")
print(f"Total Test Runs : {len(BENCHMARK_QUERIES)} Searches")
print("=" * 80)
total_accuracy_score = 0
total_searches = len(BENCHMARK_QUERIES)
for item in BENCHMARK_QUERIES:
q_id = item["id"]
query = item["query"]
print(f"\n[SEARCH #{q_id}/{total_searches}] Query: \"{query}\"")
print(f" Topic: {item['topic']} | Expected: {item['famous_ref']}")
results, t_emb, t_atl, t_rer = await run_hadith_search(query, top_k=3)
print(f" Timing: Embed={t_emb:.3f}s | Atlas={t_atl:.3f}s | Rerank={t_rer:.3f}s | Total={t_emb+t_atl+t_rer:.3f}s")
if not results:
print(" ❌ ZERO RESULTS RETURNED!")
continue
top_hit = results[0]
ref = top_hit.get("reference") or f"{top_hit.get('collection')} #{top_hit.get('hadith_no')}"
narrator = top_hit.get("narrator", "N/A")
text_en = top_hit.get("text_english", "")[:280]
vscore = top_hit.get("_vscore", 0.0)
rscore = top_hit.get("_rerank_score", 0.0)
# Check keyword presence in top 3 hits
found_keywords = [kw for kw in item["expected_keywords"] if kw in text_en.lower()]
accuracy_match = len(found_keywords) >= 2
if accuracy_match:
total_accuracy_score += 1
status_icon = "[EXACT MATCH]"
else:
status_icon = "[PARTIAL MATCH]"
print(f" {status_icon} -> Top Rank #1:")
print(f" Reference : {ref}")
print(f" Narrator : {narrator}")
print(f" Vector Sim : {vscore:.4f} | Rerank Score: {rscore:.4f}")
print(f" Text Preview: \"{text_en}...\"")
print("\n" + "=" * 80)
print("BENCHMARK SUMMARY")
print("=" * 80)
print(f"Total Searches Executed : {total_searches}")
print(f"Total OpenRouter BGE-M3 Calls : {total_searches}")
print(f"Total OpenRouter Rerank Calls : {total_searches}")
print(f"Search Accuracy Rate : {total_accuracy_score}/{total_searches} ({(total_accuracy_score/total_searches)*100:.1f}%)")
print("=" * 80)
if __name__ == "__main__":
asyncio.run(main())
|