update the quation to work with all files

app.py

@@ -9,6 +9,7 @@ from datetime import datetime
 from huggingface_hub import HfApi  # For file persistence in Spaces
 import os
 import threading
+import glob
 
 # Load environment variables from .env file
 load_dotenv()
@@ -195,7 +196,7 @@ def save_dataset_to_space(dataset: Dict, filename: str):
 @app.get("/generate-questions")
 async def generate_questions():
     """
-    Endpoint to generate questions from the vaccine guide chunks
+    Endpoint to generate questions from all JSON files in the data folder
     """
     global generation_status
     
@@ -219,23 +220,41 @@ async def generate_questions():
             generation_status["result_file"] = None
             generation_status["error"] = None
         
-        # Load chunks
-        with open("./chunks.json", "r", encoding="utf-8") as f:
-            chunks_data = json.load(f)
+        # Load all JSON files from data folder
+        json_files = glob.glob("./data/*.json")
         
-        if chunks_data is None:
-            raise HTTPException(status_code=404, detail="Chunks file not found in any known location")
+        if not json_files:
+            raise HTTPException(status_code=404, detail="No JSON files found in data folder")
         
-        VACCINE_CHUNKS = [chunk["text"] for chunk in chunks_data]
+        all_chunks = []
+        for json_file in json_files:
+            with open(json_file, "r", encoding="utf-8") as f:
+                chunks_data = json.load(f)
+                if isinstance(chunks_data, list):
+                    # If it's a list of chunks
+                    for chunk in chunks_data:
+                        if isinstance(chunk, dict) and "text" in chunk:
+                            all_chunks.append(chunk["text"])
+                        elif isinstance(chunk, str):
+                            all_chunks.append(chunk)
+                elif isinstance(chunks_data, dict):
+                    # If it's a dict, try to extract text content
+                    if "text" in chunks_data:
+                        all_chunks.append(chunks_data["text"])
+                    elif "content" in chunks_data:
+                        all_chunks.append(chunks_data["content"])
+        
+        if not all_chunks:
+            raise HTTPException(status_code=404, detail="No text content found in JSON files")
         
         # Start generation in background thread
-        thread = threading.Thread(target=generate_questions_in_background, args=(VACCINE_CHUNKS,))
+        thread = threading.Thread(target=generate_questions_in_background, args=(all_chunks,))
         thread.daemon = True
         thread.start()
         
         return {
             "status": "started",
-            "message": "Question generation started in background",
+            "message": f"Question generation started for {len(json_files)} JSON files with {len(all_chunks)} chunks",
             "current_status": generation_status
         }
     except Exception as e: