Update rag.py

rag.py

@@ -24,9 +24,23 @@ similarity_model = SentenceTransformer('paraphrase-MiniLM-L6-v2')
 HF_DATASET_REPO = "midrees2806/unmatched_queries"
 HF_TOKEN = os.getenv("HF_TOKEN")
 
-# Load dataset (automatically using the path)
-with open('dataset/', 'r') as f:
-    dataset = json.load(f)
+# Load multiple JSON datasets
+dataset = []
+try:
+    json_files = glob.glob('datasets/*.json')
+    for file_path in json_files:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+            if isinstance(data, list):
+                for item in data:
+                    if isinstance(item, dict) and 'Question' in item and 'Answer' in item:
+                        dataset.append(item)
+                    else:
+                        print(f"Invalid entry in {file_path}: {item}")
+            else:
+                print(f"File {file_path} does not contain a list.")
+except Exception as e:
+    print(f"Error loading datasets: {e}")
 
 # Precompute embeddings
 dataset_questions = [item.get("Question", "").lower().strip() for item in dataset]