add status

app.py

@@ -4,10 +4,11 @@ import json
 from dotenv import load_dotenv
 import time
 import uuid
-from typing import List, Dict
+from typing import List, Dict, Optional
 from datetime import datetime
 from huggingface_hub import HfApi  # For file persistence in Spaces
 import os
+import threading
 
 # Load environment variables from .env file
 load_dotenv()
@@ -16,6 +17,20 @@ from langchain_google_genai import GoogleGenerativeAI
 
 app = FastAPI()
 
+# Global variables to track generation status
+generation_status = {
+    "is_running": False,
+    "start_time": None,
+    "processed_chunks": 0,
+    "total_chunks": 0,
+    "questions_generated": 0,
+    "completed": False,
+    "result_file": None,
+    "error": None
+}
+
+generation_lock = threading.Lock()
+
 def estimate_difficulty(question: str, q_type: str) -> str:
     """
     Estimate question difficulty based on type and content.
@@ -99,6 +114,10 @@ def generate_questions_for_chunk(chunk: str, chunk_id: int, model="gemini-2.0-fl
                 "validated": False
             })
         
+        # Update the global status
+        with generation_lock:
+            generation_status["questions_generated"] += len(formatted_questions)
+        
         return formatted_questions
     
     except Exception as e:
@@ -108,33 +127,60 @@ def generate_questions_for_chunk(chunk: str, chunk_id: int, model="gemini-2.0-fl
         print(f"Erreur de parsing de la réponse API pour le chunk {chunk_id}: {e}")
         return []
 
-def generate_questions_for_document(chunks: List[str]) -> Dict:
+def generate_questions_in_background(chunks: List[str]):
     """
-    Generate questions for all document chunks and structure as a scientific dataset.
+    Generate questions in a background thread and update status.
     """
-    all_questions = []
+    global generation_status
     
-    for i, chunk in enumerate(chunks):
-        print(f"Processing chunk {i+1}/{len(chunks)}...")
-        questions = generate_questions_for_chunk(chunk, i)
-        all_questions.extend(questions)
-        time.sleep(9)  # Rate limiting
-    
-    dataset = {
-        "dataset_info": {
-            "title": "Vaccine Guide Question-Answer Dataset",
-            "description": "A dataset of question-answer pairs generated from a vaccine guide for AI language model training.",
-            "version": "1.1.0",
-            "created_date": datetime.utcnow().isoformat(),
-            "source": "Guide-pratique-de-mise-en-oeuvre-du-calendrier-national-de-vaccination-2023.pdf",
-            "generated_by": "Gemini API",
-            "total_questions": len(all_questions),
-            "intended_use": "Fine-tuning medical language models for knowledge recall and reasoning"
-        },
-        "questions": all_questions
-    }
+    try:
+        all_questions = []
+        
+        with generation_lock:
+            generation_status["total_chunks"] = len(chunks)
+            generation_status["processed_chunks"] = 0
+            generation_status["questions_generated"] = 0
+        
+        for i, chunk in enumerate(chunks):
+            print(f"Processing chunk {i+1}/{len(chunks)}...")
+            questions = generate_questions_for_chunk(chunk, i)
+            all_questions.extend(questions)
+            
+            with generation_lock:
+                generation_status["processed_chunks"] = i + 1
+            
+            time.sleep(9)  # Rate limiting
+        
+        dataset = {
+            "dataset_info": {
+                "title": "Vaccine Guide Question-Answer Dataset",
+                "description": "A dataset of question-answer pairs generated from a vaccine guide for AI language model training.",
+                "version": "1.1.0",
+                "created_date": datetime.utcnow().isoformat(),
+                "source": "Guide-pratique-de-mise-en-oeuvre-du-calendrier-national-de-vaccination-2023.pdf",
+                "generated_by": "Gemini API",
+                "total_questions": len(all_questions),
+                "intended_use": "Fine-tuning medical language models for knowledge recall and reasoning"
+            },
+            "questions": all_questions
+        }
+        
+        # Save the dataset
+        filename = f"vaccine_questions_{int(time.time())}.json"
+        with open(f"./{filename}", 'w', encoding='utf-8') as f:
+            json.dump(dataset, f, indent=4, ensure_ascii=False)
+        
+        # Update status to completed
+        with generation_lock:
+            generation_status["completed"] = True
+            generation_status["is_running"] = False
+            generation_status["result_file"] = filename
     
-    return dataset
+    except Exception as e:
+        print(f"Error in background generation: {e}")
+        with generation_lock:
+            generation_status["error"] = str(e)
+            generation_status["is_running"] = False
 
 def save_dataset_to_space(dataset: Dict, filename: str):
     """
@@ -151,34 +197,63 @@ async def generate_questions():
     """
     Endpoint to generate questions from the vaccine guide chunks
     """
+    global generation_status
+    
+    # Check if generation is already running
+    with generation_lock:
+        if generation_status["is_running"]:
+            return {
+                "status": "running",
+                "message": "Generation already in progress",
+                "current_status": generation_status
+            }
+    
     try:
+        # Reset status
+        with generation_lock:
+            generation_status["is_running"] = True
+            generation_status["start_time"] = datetime.utcnow().isoformat()
+            generation_status["processed_chunks"] = 0
+            generation_status["questions_generated"] = 0
+            generation_status["completed"] = False
+            generation_status["result_file"] = None
+            generation_status["error"] = None
         
-        chunks_data = None
-   
-    
+        # Load chunks
         with open("./chunks.json", "r", encoding="utf-8") as f:
-                chunks_data = json.load(f)
-
+            chunks_data = json.load(f)
         
         if chunks_data is None:
             raise HTTPException(status_code=404, detail="Chunks file not found in any known location")
         
-        VACCINE_CHUNKS = [chunks_data[0]["text"]]
-        dataset = generate_questions_for_document(VACCINE_CHUNKS)
+        VACCINE_CHUNKS = [chunk["text"] for chunk in chunks_data]
         
-        # Save to persistent storage
-        filename = "vaccine_questions.json"
-        save_dataset_to_space(dataset, filename)
+        # Start generation in background thread
+        thread = threading.Thread(target=generate_questions_in_background, args=(VACCINE_CHUNKS,))
+        thread.daemon = True
+        thread.start()
         
         return {
-            "status": "success",
-            "message": "Questions generated successfully",
-            "dataset_info": dataset["dataset_info"],
-            "download_url": f"/download/{filename}"
+            "status": "started",
+            "message": "Question generation started in background",
+            "current_status": generation_status
         }
     except Exception as e:
+        with generation_lock:
+            generation_status["is_running"] = False
+            generation_status["error"] = str(e)
         raise HTTPException(status_code=500, detail=str(e))
 
+@app.get("/generation-status")
+async def get_generation_status():
+    """
+    Endpoint to check the current status of generation
+    """
+    with generation_lock:
+        status_copy = generation_status.copy()
+    
+    return status_copy
+
 @app.get("/download/{filename}")
 async def download_file(filename: str):
     """

index.html

@@ -132,6 +132,31 @@
         .badge-factual { background-color: #3498db; }
         .badge-conceptual { background-color: #9b59b6; }
         .badge-applied { background-color: #e67e22; }
+        
+        .progress-container {
+            margin-top: 15px;
+            background-color: #ecf0f1;
+            border-radius: 4px;
+            height: 20px;
+            overflow: hidden;
+        }
+        .progress-bar {
+            height: 100%;
+            background-color: #3498db;
+            transition: width 0.3s ease;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            color: white;
+            font-size: 12px;
+            font-weight: bold;
+        }
+        .progress-bar.complete {
+            background-color: #27ae60;
+        }
+        .progress-bar.error {
+            background-color: #e74c3c;
+        }
     </style>
 </head>
 <body>
@@ -143,8 +168,19 @@
         <button id="generateBtn">Générer des Questions</button>
         
         <div id="statusContainer" style="display: none;">
-            <div class="loader"></div>
-            <span id="statusText">Initialisation de la génération...</span>
+            <div id="statusHeader">
+                <div class="loader"></div>
+                <span id="statusText">Initialisation de la génération...</span>
+            </div>
+            
+            <div class="progress-container">
+                <div id="progressBar" class="progress-bar" style="width: 0%">0%</div>
+            </div>
+            
+            <div id="statusDetails" style="margin-top: 10px;">
+                <div>Chunks traités: <span id="processedChunks">0</span>/<span id="totalChunks">0</span></div>
+                <div>Questions générées: <span id="questionsGenerated">0</span></div>
+            </div>
         </div>
     </div>
     
@@ -181,23 +217,44 @@
         const apiBaseUrl = window.location.origin; // Use the same origin for API calls
         let generatedDataset = null;
         let downloadUrl = '';
+        let statusCheckInterval = null;
         
         document.getElementById('generateBtn').addEventListener('click', startGeneration);
         document.getElementById('downloadBtn').addEventListener('click', downloadDataset);
         
+        // Check for ongoing generation when the page loads
+        window.addEventListener('load', checkOngoingGeneration);
+        
+        function checkOngoingGeneration() {
+            fetch(`${apiBaseUrl}/generation-status`)
+                .then(response => response.json())
+                .then(status => {
+                    if (status.is_running || status.completed) {
+                        setupStatusMonitoring();
+                        updateStatusDisplay(status);
+                    }
+                    
+                    if (status.completed && status.result_file) {
+                        downloadUrl = `/download/${status.result_file}`;
+                        loadResults();
+                    }
+                })
+                .catch(error => {
+                    console.error('Error checking generation status:', error);
+                });
+        }
+        
         function startGeneration() {
             const generateBtn = document.getElementById('generateBtn');
             const statusContainer = document.getElementById('statusContainer');
-            const statusText = document.getElementById('statusText');
             const resultsContainer = document.getElementById('resultsContainer');
             
             // Disable button and show status
             generateBtn.disabled = true;
             statusContainer.style.display = 'block';
-            statusText.textContent = 'Génération des questions en cours... Cela peut prendre quelques minutes.';
             resultsContainer.style.display = 'none';
             
-            // Call the API endpoint
+            // Call the API endpoint to start generation
             fetch(`${apiBaseUrl}/generate-questions`)
                 .then(response => {
                     if (!response.ok) {
@@ -206,28 +263,111 @@
                     return response.json();
                 })
                 .then(data => {
-                    console.log('Generation successful:', data);
-                    downloadUrl = data.download_url;
+                    console.log('Generation started:', data);
                     
-                    // Fetch the generated dataset
-                    return fetch(`${apiBaseUrl}${downloadUrl}`);
+                    // Setup status monitoring
+                    setupStatusMonitoring();
                 })
+                .catch(error => {
+                    console.error('Error starting generation:', error);
+                    document.getElementById('statusText').textContent = `Erreur: ${error.message}`;
+                    generateBtn.disabled = false;
+                });
+        }
+        
+        function setupStatusMonitoring() {
+            // Clear any existing interval
+            if (statusCheckInterval) {
+                clearInterval(statusCheckInterval);
+            }
+            
+            // Show the status container
+            document.getElementById('statusContainer').style.display = 'block';
+            
+            // Start checking status regularly
+            statusCheckInterval = setInterval(checkGenerationStatus, 5000);
+            
+            // Do an immediate check
+            checkGenerationStatus();
+        }
+        
+        function checkGenerationStatus() {
+            fetch(`${apiBaseUrl}/generation-status`)
                 .then(response => response.json())
+                .then(status => {
+                    updateStatusDisplay(status);
+                    
+                    // If generation is completed, stop checking and load results
+                    if (status.completed) {
+                        clearInterval(statusCheckInterval);
+                        downloadUrl = `/download/${status.result_file}`;
+                        loadResults();
+                    }
+                    
+                    // If there was an error, stop checking
+                    if (status.error) {
+                        clearInterval(statusCheckInterval);
+                        document.getElementById('generateBtn').disabled = false;
+                    }
+                })
+                .catch(error => {
+                    console.error('Error checking status:', error);
+                });
+        }
+        
+        function updateStatusDisplay(status) {
+            const statusText = document.getElementById('statusText');
+            const progressBar = document.getElementById('progressBar');
+            const processedChunks = document.getElementById('processedChunks');
+            const totalChunks = document.getElementById('totalChunks');
+            const questionsGenerated = document.getElementById('questionsGenerated');
+            
+            // Update text and counts
+            if (status.error) {
+                statusText.textContent = `Erreur: ${status.error}`;
+                progressBar.classList.add('error');
+            } else if (status.completed) {
+                statusText.textContent = 'Génération terminée avec succès !';
+                progressBar.classList.add('complete');
+            } else if (status.is_running) {
+                statusText.textContent = 'Génération des questions en cours...';
+            }
+            
+            // Update progress data
+            processedChunks.textContent = status.processed_chunks;
+            totalChunks.textContent = status.total_chunks;
+            questionsGenerated.textContent = status.questions_generated;
+            
+            // Calculate and update progress percentage
+            if (status.total_chunks > 0) {
+                const percentage = Math.round((status.processed_chunks / status.total_chunks) * 100);
+                progressBar.style.width = `${percentage}%`;
+                progressBar.textContent = `${percentage}%`;
+            }
+        }
+        
+        function loadResults() {
+            if (!downloadUrl) return;
+            
+            // Fetch the generated dataset
+            fetch(`${apiBaseUrl}${downloadUrl}`)
+                .then(response => {
+                    if (!response.ok) {
+                        throw new Error(`HTTP error! Status: ${response.status}`);
+                    }
+                    return response.json();
+                })
                 .then(dataset => {
                     generatedDataset = dataset;
                     displayResults(dataset);
                     
-                    // Update status and re-enable button
+                    // Update status
+                    const statusText = document.getElementById('statusText');
                     statusText.textContent = 'Génération terminée avec succès !';
-                    setTimeout(() => {
-                        statusContainer.style.display = 'none';
-                        generateBtn.disabled = false;
-                    }, 3000);
+                    document.getElementById('generateBtn').disabled = false;
                 })
                 .catch(error => {
-                    console.error('Error generating questions:', error);
-                    statusText.textContent = `Erreur: ${error.message}`;
-                    generateBtn.disabled = false;
+                    console.error('Error loading results:', error);
                 });
         }