Update app/data_loader.py

app/data_loader.py

@@ -1,34 +1,43 @@
 import pandas as pd
 from sentence_transformers import SentenceTransformer
 import torch
-from app.utils import remove_numbers
 from app.qdrant_client import client
 from qdrant_client.http import models
 from pympler import asizeof
+
 print("Loading model and data...")
 
+# --- Setup device ---
 device = 'cuda' if torch.cuda.is_available() else 'cpu'
 print(f"Using device: {device}")
 
+# --- Load model ---
 model = SentenceTransformer("app/my_finetuned_modelV2", device=device)
-print("model size : ", asizeof.asizeof(model))
-df = pd.read_csv("app/data/cleaned_fileV2.csv")
+print("Model size:", asizeof.asizeof(model))
+
+# --- Load ayahs from ayas.csv ---
+df = pd.read_csv("app/data/ayas.csv", encoding="utf-8")
 
-df['answer'] = df['answer'].apply(remove_numbers)
-ayat = list(set(df['answer']))
-print(f"Total unique ayat loaded: {asizeof.asizeof(ayat)}") 
+# Ensure expected columns
+if not {'answers', 'arabic'}.issubset(df.columns):
+    raise ValueError("❌ 'ayas.csv' must contain 'answers' and 'arabic' columns.")
 
-print("✅ Model and embeddings ready.")
+# Remove duplicates and NaN
+df = df.dropna(subset=['answers', 'arabic']).drop_duplicates(subset=['answers'])
+ayat = df['answers'].tolist()
+print(f"Total unique ayat loaded: {len(ayat)}")
 
+print("✅ Model and data ready.")
 
 # --- Check if collection exists ---
 collections = [c.name for c in client.get_collections().collections]
 if "ayahs_collection" not in collections:
     print("Creating Qdrant collection and uploading embeddings...")
-    embeddings = model.encode(ayat, convert_to_tensor=False).tolist()
+
+    embeddings = model.encode(ayat, convert_to_tensor=False, show_progress_bar=True).tolist()
+
     client.recreate_collection(
         collection_name="ayahs_collection",
-        
         vectors_config=models.VectorParams(
             size=len(embeddings[0]),
             distance=models.Distance.COSINE
@@ -39,7 +48,10 @@ if "ayahs_collection" not in collections:
         models.PointStruct(
             id=idx,
             vector=emb,
-            payload={"text": ayah}
+            payload={
+                "text": ayah,
+                "arabic": df.iloc[idx]['arabic']
+            }
         )
         for idx, (emb, ayah) in enumerate(zip(embeddings, ayat))
     ]
@@ -48,4 +60,3 @@ if "ayahs_collection" not in collections:
     print("✅ Embeddings uploaded to Qdrant.")
 else:
     print("✅ Collection already exists, skipping upload.")
-    # Load embeddings from Qdrant