udp app.py

app.py

@@ -1,11 +1,11 @@
 import gradio as gr
 from huggingface_hub import InferenceClient
-from datasets import load_dataset
 import faiss
 import numpy as np
 import os
 import time
-import json 
+import json
+import threading  # ✅ Run embeddings in parallel
 
 # ✅ Ensure FAISS is installed
 os.system("pip install faiss-cpu")
@@ -13,11 +13,10 @@ os.system("pip install faiss-cpu")
 def log(message):
     print(f"✅ {message}")
 
-
 DATA_DIR = "data"
 os.makedirs(DATA_DIR, exist_ok=True)  # Ensure directory exists
 
-# ✅ List of datasets
+# ✅ Step 1: Load Datasets from HF and Save Locally
 datasets = {
     "sales": "goendalf666/sales-conversations",
     "blended": "blended_skill_talk",
@@ -25,46 +24,44 @@ datasets = {
     "multiwoz": "multi_woz_v22",
 }
 
-# ✅ Save datasets to JSON
 for name, hf_name in datasets.items():
-    print(f"📥 Downloading {name} dataset...")
-    dataset = load_dataset(hf_name)
+    file_path = os.path.join(DATA_DIR, f"{name}.json")
 
-    # Extract training data
-    train_data = dataset["train"]
+    if os.path.exists(file_path):
+        log(f"✅ {name} dataset already stored at {file_path}")
+        continue  # Skip if dataset exists
 
-    # Convert dataset to list of dictionaries
+    log(f"📥 Downloading {name} dataset...")
+    dataset = load_dataset(hf_name)
+    train_data = dataset["train"]
     data_list = [dict(row) for row in train_data]
 
-    # Save to JSON
-    file_path = os.path.join(DATA_DIR, f"{name}.json")
     with open(file_path, "w") as f:
         json.dump(data_list, f, indent=2)
 
-    print(f"✅ {name} dataset saved to {file_path}")
-
-
+    log(f"✅ {name} dataset saved to {file_path}")
 
-# ✅ Step 1: Run Embedding Script (Import and Run)
-log("🚀 Running embeddings script...")
-import embeddings  # This will automatically run embeddings.py
+# ✅ Step 2: Run Embeddings in a Separate Thread
+def run_embeddings():
+    log("🚀 Running embeddings script in background...")
+    import embeddings  # ✅ This will automatically run embeddings.py
+    log("✅ Embeddings process finished.")
 
-time.sleep(5)  # Wait for embeddings to be created
+embedding_thread = threading.Thread(target=run_embeddings)
+embedding_thread.start()  # ✅ Start embedding in background
 
-# ✅ Step 2: Check FAISS index
+# ✅ Step 3: Check FAISS index
 def check_faiss():
-    index_path = "my_embeddings"  # Adjust if needed
+    index_path = "my_embeddings.faiss"  # Ensure file has .faiss extension
+
+    if not os.path.exists(index_path):
+        return "⚠️ No FAISS index found! Embeddings might still be processing."
 
     try:
         index = faiss.read_index(index_path)
         num_vectors = index.ntotal
         dim = index.d
-
-        if num_vectors > 0:
-            return f"📊 FAISS index contains {num_vectors} vectors.\n✅ Embedding dimension: {dim}"
-        else:
-            return "⚠️ No embeddings found in FAISS index!"
-
+        return f"📊 FAISS index contains {num_vectors} vectors.\n✅ Embedding dimension: {dim}"
     except Exception as e:
         return f"❌ ERROR: Failed to load FAISS index - {e}"
 
@@ -72,7 +69,7 @@ log("🔍 Checking FAISS embeddings...")
 faiss_status = check_faiss()
 log(faiss_status)
 
-# ✅ Step 3: Initialize chatbot
+# ✅ Step 4: Initialize Chatbot
 client = InferenceClient("mistralai/Mistral-7B-Instruct-v0.3")
 
 def respond(message, history, system_message, max_tokens, temperature, top_p):
@@ -94,7 +91,7 @@ def respond(message, history, system_message, max_tokens, temperature, top_p):
         response += token
         yield response
 
-# ✅ Step 4: Start Chatbot Interface
+# ✅ Step 5: Start Chatbot Interface
 demo = gr.ChatInterface(
     respond,
     additional_inputs=[
@@ -107,5 +104,5 @@ demo = gr.ChatInterface(
 
 log("✅ All systems go! Launching chatbot...")
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch()  # ✅ FIXED typo