Upload folder using huggingface_hub

.gitattributes

@@ -1,35 +1,3 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.db filter=lfs diff=lfs merge=lfs -text
+qdrant_storage/** filter=lfs diff=lfs merge=lfs -text
+*.sqlite filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,12 +1,22 @@
 ---
-title: Bayan Search
-emoji: 🏃
-colorFrom: indigo
+title: Bayan Usuli BERT API
+emoji: 📚
+colorFrom: green
 colorTo: blue
 sdk: gradio
-sdk_version: 6.0.2
+sdk_version: 4.44.0
 app_file: app.py
 pinned: false
+license: mit
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Bayan Usuli BERT API
+
+Arabic Islamic Jurisprudence (Usul al-Fiqh) Embedding Model API.
+
+This space provides an API for the `MohJaf/bayan-usuli-bert` sentence-transformers model.
+
+## Features
+
+- Get text embeddings for Arabic jurisprudence texts
+- Compute semantic similarity between texts

app.py

@@ -0,0 +1,162 @@
+import gradio as gr
+import numpy as np
+import sqlite3
+import os
+import shutil
+from sentence_transformers import SentenceTransformer
+from qdrant_client import QdrantClient
+from huggingface_hub import hf_hub_download, snapshot_download
+
+# Configuration
+MODEL_PATH = "MohJaf/bayan-usuli-bert"
+DATASET_REPO = "MohJaf/bayan-usuli-data"
+QDRANT_PATH = "./qdrant_storage"
+SQLITE_PATH = "./usuli_library.db"
+
+def download_data():
+    """Download database files from HuggingFace Dataset"""
+    print("Downloading data from HuggingFace...")
+
+    # Download SQLite database
+    if not os.path.exists(SQLITE_PATH):
+        print("Downloading SQLite database...")
+        sqlite_file = hf_hub_download(
+            repo_id=DATASET_REPO,
+            filename="usuli_library.db",
+            repo_type="dataset"
+        )
+        shutil.copy(sqlite_file, SQLITE_PATH)
+        print(f"SQLite database downloaded to {SQLITE_PATH}")
+
+    # Download Qdrant storage
+    if not os.path.exists(QDRANT_PATH):
+        print("Downloading Qdrant storage...")
+        # Download the entire qdrant_storage folder
+        snapshot_path = snapshot_download(
+            repo_id=DATASET_REPO,
+            repo_type="dataset",
+            allow_patterns=["qdrant_storage/*"]
+        )
+        # Copy to local path
+        src_qdrant = os.path.join(snapshot_path, "qdrant_storage")
+        if os.path.exists(src_qdrant):
+            shutil.copytree(src_qdrant, QDRANT_PATH)
+            print(f"Qdrant storage downloaded to {QDRANT_PATH}")
+
+    print("Data download complete!")
+
+# Download data first
+download_data()
+
+# Load model and databases
+print("Loading model...")
+model = SentenceTransformer(MODEL_PATH)
+print("Model loaded!")
+
+print("Connecting to Qdrant...")
+qdrant = QdrantClient(path=QDRANT_PATH)
+print("Qdrant connected!")
+
+def get_card_details(card_ids):
+    """Fetch card details from SQLite"""
+    if not card_ids:
+        return {}
+
+    conn = sqlite3.connect(SQLITE_PATH)
+    cursor = conn.cursor()
+
+    placeholders = ",".join(["?" for _ in card_ids])
+    query = f"""
+        SELECT ac.id, b.title, ac.full_text, ac.paragraph_id, ac.page_start
+        FROM atomized_cards ac
+        JOIN books b ON b.id = ac.book_id
+        WHERE ac.id IN ({placeholders})
+    """
+
+    cursor.execute(query, card_ids)
+    rows = cursor.fetchall()
+    conn.close()
+
+    results = {}
+    for row in rows:
+        card_id, title, text, para_id, page = row
+        results[card_id] = {
+            "title": title,
+            "text": text,
+            "reference": para_id or f"p{page}"
+        }
+    return results
+
+def search(query, top_k=5):
+    """Semantic search for usuli texts"""
+    if not query or not query.strip():
+        return {"error": "No query provided"}
+
+    # Encode query
+    query_vector = model.encode(query).tolist()
+
+    # Search in Qdrant
+    results = qdrant.search(
+        collection_name="usuli_cards",
+        query_vector=query_vector,
+        limit=int(top_k),
+        with_payload=True
+    )
+
+    # Get card IDs
+    card_ids = [r.payload.get("id") for r in results if r.payload]
+
+    # Fetch details from SQLite
+    card_details = get_card_details(card_ids)
+
+    # Format results
+    hits = []
+    for r in results:
+        card_id = r.payload.get("id") if r.payload else None
+        if card_id and card_id in card_details:
+            details = card_details[card_id]
+            hits.append({
+                "title": details["title"],
+                "text": details["text"][:500] + "..." if len(details["text"]) > 500 else details["text"],
+                "reference": details["reference"],
+                "score": round(r.score, 4)
+            })
+
+    # Build answer
+    if hits:
+        answer_parts = []
+        for h in hits[:3]:
+            answer_parts.append(f"{h['title']} ({h['reference']}): {h['text'][:200]}...")
+        answer = "\n\n".join(answer_parts)
+    else:
+        answer = "لم يتم العثور على نتائج"
+
+    return {
+        "query": query,
+        "answer": answer,
+        "results": hits
+    }
+
+# Gradio interface
+with gr.Blocks(title="Bayan Usuli Search API", theme=gr.themes.Soft()) as demo:
+    gr.Markdown("# بيان - البحث الدلالي في علم الأصول")
+    gr.Markdown("ابحث في النصوص الأصولية باستخدام الذكاء الاصطناعي")
+
+    with gr.Row():
+        query_input = gr.Textbox(
+            label="السؤال",
+            placeholder="اكتب سؤالك الأصولي هنا...",
+            rtl=True,
+            lines=2
+        )
+
+    with gr.Row():
+        top_k = gr.Slider(minimum=1, maximum=10, value=5, step=1, label="عدد النتائج")
+        search_btn = gr.Button("بحث", variant="primary")
+
+    output = gr.JSON(label="النتائج")
+
+    search_btn.click(search, inputs=[query_input, top_k], outputs=output)
+    query_input.submit(search, inputs=[query_input, top_k], outputs=output)
+
+demo.launch()

qdrant_storage/.lock

@@ -0,0 +1 @@
+tmp lock file

qdrant_storage/collection/usuli_cards/storage.sqlite

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6738bad99786d8934b5fb34b9b3bcade57fabb6d5b53b3203d528fc2532af569
+size 542756864

qdrant_storage/meta.json

@@ -0,0 +1 @@
+{"collections": {"usuli_cards": {"vectors": {"size": 768, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}}, "aliases": {}}

requirements.txt

@@ -0,0 +1,6 @@
+gradio>=4.0.0
+sentence-transformers>=2.2.0
+qdrant-client>=1.7.0
+huggingface_hub>=0.20.0
+torch
+numpy

usuli_library.db

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c391b99bf14a65a5e08811d1c2d11ca11df96826765d5e25a444a6d4153b3ff1
+size 143626240