Zeggai Abdellah commited on
Commit
3552aba
·
1 Parent(s): 659455b

update the app.py and index.html to support mulit api key

Browse files
Files changed (2) hide show
  1. app.py +306 -58
  2. index.html +493 -218
app.py CHANGED
@@ -10,12 +10,12 @@ from huggingface_hub import HfApi # For file persistence in Spaces
10
  import os
11
  import threading
12
  import glob
 
 
13
 
14
  # Load environment variables from .env file
15
  load_dotenv()
16
 
17
- from langchain_google_genai import GoogleGenerativeAI
18
-
19
  app = FastAPI()
20
 
21
  # Global variables to track generation status
@@ -27,11 +27,104 @@ generation_status = {
27
  "questions_generated": 0,
28
  "completed": False,
29
  "result_file": None,
30
- "error": None
 
 
 
 
31
  }
32
 
33
  generation_lock = threading.Lock()
34
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
35
  def estimate_difficulty(question: str, q_type: str) -> str:
36
  """
37
  Estimate question difficulty based on type and content.
@@ -49,9 +142,10 @@ def estimate_difficulty(question: str, q_type: str) -> str:
49
  return "medium"
50
  return "hard" # applied
51
 
52
- def generate_questions_for_chunk(chunk: str, chunk_id: int, model="gemini-2.0-flash") -> List[Dict]:
53
  """
54
  Generate French questions for a given document chunk using the Gemini API.
 
55
  """
56
  prompt = f"""
57
  À partir du texte suivant d'un guide sur les vaccins en français, g��nérez 3 questions variées (factual, conceptual, applied) qui couvrent le contenu de manière exhaustive.
@@ -78,59 +172,91 @@ def generate_questions_for_chunk(chunk: str, chunk_id: int, model="gemini-2.0-fl
78
  ```
79
  """
80
 
81
- try:
82
- llm = GoogleGenerativeAI(
83
- model=model,
84
- google_api_key=os.getenv("GOOGLE_API_KEY")
85
- )
 
 
 
 
 
 
 
86
 
87
- response = llm.invoke(prompt)
88
-
89
- questions_text = str(response) # Convert response to string
90
-
91
- # Strip Markdown code fences
92
- if questions_text.startswith("```json\n") and questions_text.endswith("\n```"):
93
- questions_text = questions_text[7:-4].strip()
94
- elif questions_text.startswith("```") and questions_text.endswith("```"):
95
- questions_text = questions_text[3:-3].strip()
96
-
97
- if not questions_text:
98
- print(f"Erreur: Réponse vide pour le chunk {chunk_id}")
99
- return []
100
-
101
- questions = json.loads(questions_text)
102
 
103
- formatted_questions = []
104
- for q in questions:
105
- question_id = str(uuid.uuid4())
106
- difficulty = estimate_difficulty(q["question"], q["type"])
107
- formatted_questions.append({
108
- "question_id": question_id,
109
- "chunk_id": chunk_id,
110
- "chunk_text": chunk,
111
- "question": q["question"],
112
- "type": q["type"],
113
- "difficulty": difficulty,
114
- "training_purpose": "Knowledge Recall" if q["type"] == "factual" else "Reasoning",
115
- "validated": False
116
- })
117
-
118
- # Update the global status
119
- with generation_lock:
120
- generation_status["questions_generated"] += len(formatted_questions)
121
-
122
- return formatted_questions
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
123
 
124
- except Exception as e:
125
- print(f"Erreur lors de la génération des questions pour le chunk {chunk_id}: {e}")
126
- return []
127
- except json.JSONDecodeError as e:
128
- print(f"Erreur de parsing de la réponse API pour le chunk {chunk_id}: {e}")
129
- return []
 
 
 
 
 
 
130
 
131
  def generate_questions_in_background(chunks: List[str]):
132
  """
133
  Generate questions in a background thread and update status.
 
134
  """
135
  global generation_status
136
 
@@ -141,17 +267,27 @@ def generate_questions_in_background(chunks: List[str]):
141
  generation_status["total_chunks"] = len(chunks)
142
  generation_status["processed_chunks"] = 0
143
  generation_status["questions_generated"] = 0
 
 
 
 
 
144
 
145
  for i, chunk in enumerate(chunks):
146
  print(f"Processing chunk {i+1}/{len(chunks)}...")
147
  questions = generate_questions_for_chunk(chunk, i)
148
- all_questions.extend(questions)
 
 
149
 
150
  with generation_lock:
151
  generation_status["processed_chunks"] = i + 1
152
 
153
- time.sleep(9) # Rate limiting
 
 
154
 
 
155
  dataset = {
156
  "dataset_info": {
157
  "title": "Vaccine Guide Question-Answer Dataset",
@@ -161,13 +297,17 @@ def generate_questions_in_background(chunks: List[str]):
161
  "source": "Guide-pratique-de-mise-en-oeuvre-du-calendrier-national-de-vaccination-2023.pdf",
162
  "generated_by": "Gemini API",
163
  "total_questions": len(all_questions),
164
- "intended_use": "Fine-tuning medical language models for knowledge recall and reasoning"
 
 
 
 
165
  },
166
  "questions": all_questions
167
  }
168
 
169
- # Save the dataset
170
- filename = f"vaccine_questions_{int(time.time())}.json"
171
  with open(f"./{filename}", 'w', encoding='utf-8') as f:
172
  json.dump(dataset, f, indent=4, ensure_ascii=False)
173
 
@@ -176,12 +316,21 @@ def generate_questions_in_background(chunks: List[str]):
176
  generation_status["completed"] = True
177
  generation_status["is_running"] = False
178
  generation_status["result_file"] = filename
 
 
 
 
 
 
179
 
180
  except Exception as e:
181
  print(f"Error in background generation: {e}")
182
  with generation_lock:
183
  generation_status["error"] = str(e)
184
  generation_status["is_running"] = False
 
 
 
185
 
186
  def save_dataset_to_space(dataset: Dict, filename: str):
187
  """
@@ -191,12 +340,12 @@ def save_dataset_to_space(dataset: Dict, filename: str):
191
  with open(persistent_path, 'w', encoding='utf-8') as f:
192
  json.dump(dataset, f, indent=4, ensure_ascii=False)
193
  print(f"Dataset saved to {persistent_path}")
194
-
195
 
196
  @app.get("/generate-questions")
197
  async def generate_questions():
198
  """
199
  Endpoint to generate questions from all JSON files in the data folder
 
200
  """
201
  global generation_status
202
 
@@ -210,6 +359,10 @@ async def generate_questions():
210
  }
211
 
212
  try:
 
 
 
 
213
  # Reset status
214
  with generation_lock:
215
  generation_status["is_running"] = True
@@ -218,13 +371,17 @@ async def generate_questions():
218
  generation_status["questions_generated"] = 0
219
  generation_status["completed"] = False
220
  generation_status["result_file"] = None
 
221
  generation_status["error"] = None
 
 
 
222
 
223
  # Load all JSON files from data folder
224
  json_files = glob.glob("./chunk/*.json")
225
 
226
  if not json_files:
227
- raise HTTPException(status_code=404, detail="No JSON files found in data folder")
228
 
229
  all_chunks = []
230
  for json_file in json_files:
@@ -255,6 +412,7 @@ async def generate_questions():
255
  return {
256
  "status": "started",
257
  "message": f"Question generation started for {len(json_files)} JSON files with {len(all_chunks)} chunks",
 
258
  "current_status": generation_status
259
  }
260
  except Exception as e:
@@ -267,21 +425,111 @@ async def generate_questions():
267
  async def get_generation_status():
268
  """
269
  Endpoint to check the current status of generation
 
270
  """
271
  with generation_lock:
272
  status_copy = generation_status.copy()
273
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
274
  return status_copy
275
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
276
  @app.get("/download/{filename}")
277
  async def download_file(filename: str):
278
  """
279
  Endpoint to download generated files
 
280
  """
281
  file_path = f"./{filename}"
282
  if os.path.exists(file_path):
283
  return FileResponse(file_path, media_type="application/json", filename=filename)
284
- raise HTTPException(status_code=404, detail="File not found")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
285
 
286
  @app.get("/")
287
  async def root():
 
10
  import os
11
  import threading
12
  import glob
13
+ import random
14
+ from langchain_google_genai import GoogleGenerativeAI
15
 
16
  # Load environment variables from .env file
17
  load_dotenv()
18
 
 
 
19
  app = FastAPI()
20
 
21
  # Global variables to track generation status
 
27
  "questions_generated": 0,
28
  "completed": False,
29
  "result_file": None,
30
+ "progress_file": None, # New: track progress file
31
+ "error": None,
32
+ "current_api_key_index": 0, # New: track current API key
33
+ "failed_chunks": [], # New: track failed chunks for retry
34
+ "partial_results": [] # New: store partial results
35
  }
36
 
37
  generation_lock = threading.Lock()
38
 
39
+ def get_api_keys() -> List[str]:
40
+ """
41
+ Get all available Google API keys from environment variables.
42
+ Supports GOOGLE_API_KEY, GOOGLE_API_KEY_1, GOOGLE_API_KEY_2, etc.
43
+ """
44
+ api_keys = []
45
+
46
+ # Check for primary key
47
+ primary_key = os.getenv("GOOGLE_API_KEY")
48
+ if primary_key:
49
+ api_keys.append(primary_key)
50
+
51
+ # Check for numbered keys
52
+ i = 1
53
+ while True:
54
+ key = os.getenv(f"GOOGLE_API_KEY_{i}")
55
+ if key:
56
+ api_keys.append(key)
57
+ i += 1
58
+ else:
59
+ break
60
+
61
+ if not api_keys:
62
+ raise ValueError("No Google API keys found in environment variables")
63
+
64
+ return api_keys
65
+
66
+ def get_next_api_key() -> tuple[str, int]:
67
+ """
68
+ Get the next API key in rotation and update the current index.
69
+ Returns tuple of (api_key, key_index)
70
+ """
71
+ global generation_status
72
+
73
+ api_keys = get_api_keys()
74
+
75
+ with generation_lock:
76
+ current_index = generation_status["current_api_key_index"]
77
+ next_index = (current_index + 1) % len(api_keys)
78
+ generation_status["current_api_key_index"] = next_index
79
+
80
+ return api_keys[next_index], next_index
81
+
82
+ def save_progress_file():
83
+ """
84
+ Save current progress to a file that can be downloaded at any time.
85
+ """
86
+ global generation_status
87
+
88
+ with generation_lock:
89
+ progress_data = {
90
+ "generation_info": {
91
+ "status": "in_progress" if generation_status["is_running"] else "completed",
92
+ "start_time": generation_status["start_time"],
93
+ "processed_chunks": generation_status["processed_chunks"],
94
+ "total_chunks": generation_status["total_chunks"],
95
+ "questions_generated": generation_status["questions_generated"],
96
+ "completed": generation_status["completed"],
97
+ "current_time": datetime.utcnow().isoformat(),
98
+ "failed_chunks": generation_status["failed_chunks"].copy(),
99
+ "error": generation_status["error"]
100
+ },
101
+ "partial_dataset": {
102
+ "dataset_info": {
103
+ "title": "Vaccine Guide Question-Answer Dataset (Partial)",
104
+ "description": "Partial dataset of question-answer pairs generated from a vaccine guide.",
105
+ "version": "1.1.0",
106
+ "created_date": generation_status["start_time"],
107
+ "source": "Guide-pratique-de-mise-en-oeuvre-du-calendrier-national-de-vaccination-2023.pdf",
108
+ "generated_by": "Gemini API",
109
+ "total_questions": len(generation_status["partial_results"]),
110
+ "intended_use": "Fine-tuning medical language models for knowledge recall and reasoning",
111
+ "note": "This is a partial dataset. Generation may still be in progress."
112
+ },
113
+ "questions": generation_status["partial_results"].copy()
114
+ }
115
+ }
116
+
117
+ # Save progress file
118
+ progress_filename = f"vaccine_questions_progress_{int(time.time())}.json"
119
+ generation_status["progress_file"] = progress_filename
120
+
121
+ try:
122
+ with open(f"./{progress_filename}", 'w', encoding='utf-8') as f:
123
+ json.dump(progress_data, f, indent=4, ensure_ascii=False)
124
+ print(f"Progress saved to {progress_filename}")
125
+ except Exception as e:
126
+ print(f"Error saving progress file: {e}")
127
+
128
  def estimate_difficulty(question: str, q_type: str) -> str:
129
  """
130
  Estimate question difficulty based on type and content.
 
142
  return "medium"
143
  return "hard" # applied
144
 
145
+ def generate_questions_for_chunk(chunk: str, chunk_id: int, model="gemini-2.0-flash", max_retries=3) -> List[Dict]:
146
  """
147
  Generate French questions for a given document chunk using the Gemini API.
148
+ Now includes retry logic with different API keys.
149
  """
150
  prompt = f"""
151
  À partir du texte suivant d'un guide sur les vaccins en français, g��nérez 3 questions variées (factual, conceptual, applied) qui couvrent le contenu de manière exhaustive.
 
172
  ```
173
  """
174
 
175
+ last_error = None
176
+
177
+ for attempt in range(max_retries):
178
+ try:
179
+ # Get next API key for this attempt
180
+ api_key, key_index = get_next_api_key()
181
+ print(f"Chunk {chunk_id}, attempt {attempt + 1}: Using API key index {key_index}")
182
+
183
+ llm = GoogleGenerativeAI(
184
+ model=model,
185
+ google_api_key=api_key
186
+ )
187
 
188
+ response = llm.invoke(prompt)
189
+
190
+ questions_text = str(response) # Convert response to string
191
+
192
+ # Strip Markdown code fences
193
+ if questions_text.startswith("```json\n") and questions_text.endswith("\n```"):
194
+ questions_text = questions_text[7:-4].strip()
195
+ elif questions_text.startswith("```") and questions_text.endswith("```"):
196
+ questions_text = questions_text[3:-3].strip()
197
+
198
+ if not questions_text:
199
+ raise ValueError(f"Empty response for chunk {chunk_id}")
200
+
201
+ questions = json.loads(questions_text)
 
202
 
203
+ formatted_questions = []
204
+ for q in questions:
205
+ question_id = str(uuid.uuid4())
206
+ difficulty = estimate_difficulty(q["question"], q["type"])
207
+ formatted_questions.append({
208
+ "question_id": question_id,
209
+ "chunk_id": chunk_id,
210
+ "chunk_text": chunk,
211
+ "question": q["question"],
212
+ "type": q["type"],
213
+ "difficulty": difficulty,
214
+ "training_purpose": "Knowledge Recall" if q["type"] == "factual" else "Reasoning",
215
+ "validated": False,
216
+ "api_key_used": key_index, # Track which key was used
217
+ "generation_attempt": attempt + 1
218
+ })
219
+
220
+ # Update the global status and add to partial results
221
+ with generation_lock:
222
+ generation_status["questions_generated"] += len(formatted_questions)
223
+ generation_status["partial_results"].extend(formatted_questions)
224
+
225
+ # Save progress after each successful chunk
226
+ save_progress_file()
227
+
228
+ print(f"Successfully generated {len(formatted_questions)} questions for chunk {chunk_id}")
229
+ return formatted_questions
230
+
231
+ except Exception as e:
232
+ last_error = e
233
+ print(f"Attempt {attempt + 1} failed for chunk {chunk_id}: {e}")
234
+
235
+ # If this is not the last attempt, wait before retrying
236
+ if attempt < max_retries - 1:
237
+ wait_time = (attempt + 1) * 5 # Increasing wait time
238
+ print(f"Waiting {wait_time} seconds before retry...")
239
+ time.sleep(wait_time)
240
+
241
+ continue
242
 
243
+ # All attempts failed
244
+ print(f"All {max_retries} attempts failed for chunk {chunk_id}. Last error: {last_error}")
245
+
246
+ # Add to failed chunks list
247
+ with generation_lock:
248
+ generation_status["failed_chunks"].append({
249
+ "chunk_id": chunk_id,
250
+ "error": str(last_error),
251
+ "attempts": max_retries
252
+ })
253
+
254
+ return []
255
 
256
  def generate_questions_in_background(chunks: List[str]):
257
  """
258
  Generate questions in a background thread and update status.
259
+ Enhanced with better error handling and progress tracking.
260
  """
261
  global generation_status
262
 
 
267
  generation_status["total_chunks"] = len(chunks)
268
  generation_status["processed_chunks"] = 0
269
  generation_status["questions_generated"] = 0
270
+ generation_status["partial_results"] = []
271
+ generation_status["failed_chunks"] = []
272
+
273
+ # Save initial progress file
274
+ save_progress_file()
275
 
276
  for i, chunk in enumerate(chunks):
277
  print(f"Processing chunk {i+1}/{len(chunks)}...")
278
  questions = generate_questions_for_chunk(chunk, i)
279
+
280
+ if questions: # Only add if generation was successful
281
+ all_questions.extend(questions)
282
 
283
  with generation_lock:
284
  generation_status["processed_chunks"] = i + 1
285
 
286
+ # Rate limiting - slightly randomized to avoid hitting limits
287
+ sleep_time = random.uniform(8, 11) # Random between 8-11 seconds
288
+ time.sleep(sleep_time)
289
 
290
+ # Create final dataset
291
  dataset = {
292
  "dataset_info": {
293
  "title": "Vaccine Guide Question-Answer Dataset",
 
297
  "source": "Guide-pratique-de-mise-en-oeuvre-du-calendrier-national-de-vaccination-2023.pdf",
298
  "generated_by": "Gemini API",
299
  "total_questions": len(all_questions),
300
+ "intended_use": "Fine-tuning medical language models for knowledge recall and reasoning",
301
+ "total_chunks_processed": len(chunks),
302
+ "successful_chunks": len(chunks) - len(generation_status["failed_chunks"]),
303
+ "failed_chunks": len(generation_status["failed_chunks"]),
304
+ "failed_chunk_details": generation_status["failed_chunks"].copy()
305
  },
306
  "questions": all_questions
307
  }
308
 
309
+ # Save the final dataset
310
+ filename = f"vaccine_questions_final_{int(time.time())}.json"
311
  with open(f"./{filename}", 'w', encoding='utf-8') as f:
312
  json.dump(dataset, f, indent=4, ensure_ascii=False)
313
 
 
316
  generation_status["completed"] = True
317
  generation_status["is_running"] = False
318
  generation_status["result_file"] = filename
319
+
320
+ # Save final progress file
321
+ save_progress_file()
322
+
323
+ success_rate = (len(chunks) - len(generation_status["failed_chunks"])) / len(chunks) * 100
324
+ print(f"Generation completed! Success rate: {success_rate:.1f}% ({len(all_questions)} questions generated)")
325
 
326
  except Exception as e:
327
  print(f"Error in background generation: {e}")
328
  with generation_lock:
329
  generation_status["error"] = str(e)
330
  generation_status["is_running"] = False
331
+
332
+ # Save progress even if there was an error
333
+ save_progress_file()
334
 
335
  def save_dataset_to_space(dataset: Dict, filename: str):
336
  """
 
340
  with open(persistent_path, 'w', encoding='utf-8') as f:
341
  json.dump(dataset, f, indent=4, ensure_ascii=False)
342
  print(f"Dataset saved to {persistent_path}")
 
343
 
344
  @app.get("/generate-questions")
345
  async def generate_questions():
346
  """
347
  Endpoint to generate questions from all JSON files in the data folder
348
+ Enhanced with multi-key support validation
349
  """
350
  global generation_status
351
 
 
359
  }
360
 
361
  try:
362
+ # Validate API keys before starting
363
+ api_keys = get_api_keys()
364
+ print(f"Found {len(api_keys)} API keys for rotation")
365
+
366
  # Reset status
367
  with generation_lock:
368
  generation_status["is_running"] = True
 
371
  generation_status["questions_generated"] = 0
372
  generation_status["completed"] = False
373
  generation_status["result_file"] = None
374
+ generation_status["progress_file"] = None
375
  generation_status["error"] = None
376
+ generation_status["current_api_key_index"] = 0
377
+ generation_status["failed_chunks"] = []
378
+ generation_status["partial_results"] = []
379
 
380
  # Load all JSON files from data folder
381
  json_files = glob.glob("./chunk/*.json")
382
 
383
  if not json_files:
384
+ raise HTTPException(status_code=404, detail="No JSON files found in chunk folder")
385
 
386
  all_chunks = []
387
  for json_file in json_files:
 
412
  return {
413
  "status": "started",
414
  "message": f"Question generation started for {len(json_files)} JSON files with {len(all_chunks)} chunks",
415
+ "api_keys_available": len(api_keys),
416
  "current_status": generation_status
417
  }
418
  except Exception as e:
 
425
  async def get_generation_status():
426
  """
427
  Endpoint to check the current status of generation
428
+ Enhanced with more detailed status information
429
  """
430
  with generation_lock:
431
  status_copy = generation_status.copy()
432
 
433
+ # Calculate additional metrics
434
+ if status_copy["total_chunks"] > 0:
435
+ progress_percentage = (status_copy["processed_chunks"] / status_copy["total_chunks"]) * 100
436
+ status_copy["progress_percentage"] = round(progress_percentage, 2)
437
+ else:
438
+ status_copy["progress_percentage"] = 0
439
+
440
+ # Add estimated time remaining if generation is running
441
+ if status_copy["is_running"] and status_copy["start_time"] and status_copy["processed_chunks"] > 0:
442
+ start_time = datetime.fromisoformat(status_copy["start_time"].replace('Z', '+00:00'))
443
+ elapsed_time = (datetime.utcnow() - start_time.replace(tzinfo=None)).total_seconds()
444
+ chunks_per_second = status_copy["processed_chunks"] / elapsed_time if elapsed_time > 0 else 0
445
+
446
+ if chunks_per_second > 0:
447
+ remaining_chunks = status_copy["total_chunks"] - status_copy["processed_chunks"]
448
+ estimated_remaining_seconds = remaining_chunks / chunks_per_second
449
+ status_copy["estimated_remaining_minutes"] = round(estimated_remaining_seconds / 60, 2)
450
+ else:
451
+ status_copy["estimated_remaining_minutes"] = None
452
+
453
  return status_copy
454
 
455
+ @app.get("/download-progress")
456
+ async def download_progress():
457
+ """
458
+ New endpoint to download current progress at any time
459
+ """
460
+ global generation_status
461
+
462
+ # Force save current progress
463
+ save_progress_file()
464
+
465
+ with generation_lock:
466
+ progress_file = generation_status["progress_file"]
467
+
468
+ if progress_file and os.path.exists(f"./{progress_file}"):
469
+ return FileResponse(f"./{progress_file}", media_type="application/json", filename=progress_file)
470
+ else:
471
+ raise HTTPException(status_code=404, detail="No progress file available")
472
+
473
  @app.get("/download/{filename}")
474
  async def download_file(filename: str):
475
  """
476
  Endpoint to download generated files
477
+ Enhanced with better error handling
478
  """
479
  file_path = f"./{filename}"
480
  if os.path.exists(file_path):
481
  return FileResponse(file_path, media_type="application/json", filename=filename)
482
+ raise HTTPException(status_code=404, detail=f"File {filename} not found")
483
+
484
+ @app.get("/retry-failed")
485
+ async def retry_failed_chunks():
486
+ """
487
+ New endpoint to retry only the failed chunks
488
+ """
489
+ global generation_status
490
+
491
+ with generation_lock:
492
+ if generation_status["is_running"]:
493
+ return {
494
+ "status": "error",
495
+ "message": "Cannot retry while generation is running"
496
+ }
497
+
498
+ failed_chunks = generation_status["failed_chunks"].copy()
499
+
500
+ if not failed_chunks:
501
+ return {
502
+ "status": "success",
503
+ "message": "No failed chunks to retry"
504
+ }
505
+
506
+ # This would require implementing the retry logic
507
+ # For now, just return the failed chunks info
508
+ return {
509
+ "status": "info",
510
+ "message": f"Found {len(failed_chunks)} failed chunks",
511
+ "failed_chunks": failed_chunks,
512
+ "note": "Retry functionality can be implemented based on requirements"
513
+ }
514
+
515
+ @app.get("/api-keys-status")
516
+ async def get_api_keys_status():
517
+ """
518
+ New endpoint to check API keys status
519
+ """
520
+ try:
521
+ api_keys = get_api_keys()
522
+ return {
523
+ "status": "success",
524
+ "total_keys": len(api_keys),
525
+ "current_key_index": generation_status["current_api_key_index"],
526
+ "message": f"{len(api_keys)} API keys configured for rotation"
527
+ }
528
+ except Exception as e:
529
+ return {
530
+ "status": "error",
531
+ "message": str(e)
532
+ }
533
 
534
  @app.get("/")
535
  async def root():
index.html CHANGED
@@ -3,64 +3,102 @@
3
  <head>
4
  <meta charset="UTF-8">
5
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
6
- <title>Générateur de Questions sur les Vaccins</title>
7
  <style>
8
  body {
9
  font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
10
  line-height: 1.6;
11
  color: #333;
12
- max-width: 1000px;
13
  margin: 0 auto;
14
  padding: 20px;
15
- background-color: #f5f7fa;
 
16
  }
17
  h1 {
18
- color: #2c3e50;
19
  text-align: center;
20
- border-bottom: 2px solid #3498db;
21
- padding-bottom: 10px;
 
22
  }
23
  .container {
24
- background-color: white;
25
- border-radius: 8px;
26
- box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
27
- padding: 20px;
 
28
  margin-bottom: 20px;
 
29
  }
30
  button {
31
- background-color: #3498db;
32
  color: white;
33
  border: none;
34
- padding: 10px 20px;
35
- border-radius: 4px;
36
  cursor: pointer;
37
  font-size: 16px;
38
- transition: background-color 0.3s;
 
 
39
  }
40
  button:hover {
41
- background-color: #2980b9;
 
42
  }
43
  button:disabled {
44
- background-color: #95a5a6;
45
  cursor: not-allowed;
 
 
46
  }
47
  .download-btn {
48
- background-color: #27ae60;
 
49
  }
50
  .download-btn:hover {
51
- background-color: #219955;
52
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
53
  #statusContainer {
54
  margin-top: 20px;
 
 
 
 
 
 
 
 
55
  padding: 15px;
56
- border-left: 4px solid #3498db;
57
- background-color: #e8f4fc;
 
58
  }
 
 
 
 
 
59
  .loader {
60
  display: inline-block;
61
  width: 20px;
62
  height: 20px;
63
- border: 3px solid rgba(0, 0, 0, 0.1);
64
  border-radius: 50%;
65
  border-top-color: #3498db;
66
  animation: spin 1s ease infinite;
@@ -70,6 +108,7 @@
70
  @keyframes spin {
71
  to { transform: rotate(360deg); }
72
  }
 
73
  .stats-container {
74
  display: grid;
75
  grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
@@ -77,14 +116,18 @@
77
  margin-top: 20px;
78
  }
79
  .stat-card {
80
- background-color: white;
81
- padding: 15px;
82
- border-radius: 8px;
83
- box-shadow: 0 2px 4px rgba(0, 0, 0, 0.05);
84
  text-align: center;
 
 
 
 
85
  }
86
  .stat-value {
87
- font-size: 24px;
88
  font-weight: bold;
89
  color: #2c3e50;
90
  margin: 10px 0;
@@ -92,80 +135,181 @@
92
  .stat-label {
93
  color: #7f8c8d;
94
  font-size: 14px;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
95
  }
 
96
  #questionsPreview {
97
  margin-top: 20px;
98
- max-height: 400px;
99
  overflow-y: auto;
100
- background-color: white;
101
- border-radius: 8px;
102
- box-shadow: 0 2px 4px rgba(0, 0, 0, 0.05);
103
  }
104
  table {
105
  width: 100%;
106
  border-collapse: collapse;
107
  }
108
  th, td {
109
- padding: 12px 15px;
110
  text-align: left;
111
- border-bottom: 1px solid #ddd;
112
  }
113
  th {
114
- background-color: #f8f9fa;
115
  font-weight: 600;
 
 
 
116
  }
117
  tr:hover {
118
- background-color: #f1f5f9;
119
  }
 
120
  .badge {
121
  display: inline-block;
122
- padding: 4px 8px;
123
- border-radius: 4px;
124
- font-size: 12px;
125
  font-weight: 600;
126
  text-transform: uppercase;
127
  color: white;
128
- }
129
- .badge-easy { background-color: #27ae60; }
130
- .badge-medium { background-color: #f39c12; }
131
- .badge-hard { background-color: #e74c3c; }
132
- .badge-factual { background-color: #3498db; }
133
- .badge-conceptual { background-color: #9b59b6; }
134
- .badge-applied { background-color: #e67e22; }
 
135
 
136
- .progress-container {
137
- margin-top: 15px;
138
- background-color: #ecf0f1;
139
- border-radius: 4px;
140
- height: 20px;
141
- overflow: hidden;
142
- }
143
- .progress-bar {
144
- height: 100%;
145
- background-color: #3498db;
146
- transition: width 0.3s ease;
147
  display: flex;
 
 
 
148
  align-items: center;
149
- justify-content: center;
150
- color: white;
151
- font-size: 12px;
152
- font-weight: bold;
153
  }
154
- .progress-bar.complete {
155
- background-color: #27ae60;
 
 
 
156
  }
157
- .progress-bar.error {
158
- background-color: #e74c3c;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
159
  }
160
  </style>
161
  </head>
162
  <body>
163
- <h1>Générateur de Questions sur les Vaccins</h1>
164
 
 
165
  <div class="container">
166
- <h2>Générer des Questions</h2>
167
- <p>Cliquez sur le bouton ci-dessous pour commencer la génération de questions à partir du guide de vaccination.</p>
168
- <button id="generateBtn">Générer des Questions</button>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
169
 
170
  <div id="statusContainer" style="display: none;">
171
  <div id="statusHeader">
@@ -177,25 +321,39 @@
177
  <div id="progressBar" class="progress-bar" style="width: 0%">0%</div>
178
  </div>
179
 
180
- <div id="statusDetails" style="margin-top: 10px;">
181
- <div>Chunks traités: <span id="processedChunks">0</span>/<span id="totalChunks">0</span></div>
182
- <div>Questions générées: <span id="questionsGenerated">0</span></div>
 
 
 
 
 
 
 
 
 
 
 
 
 
183
  </div>
184
  </div>
185
  </div>
186
 
 
187
  <div class="container" id="resultsContainer" style="display: none;">
188
- <h2>Résultats</h2>
189
 
190
  <div class="stats-container" id="statsContainer">
191
  <!-- Stats will be populated here -->
192
  </div>
193
 
194
- <div style="margin-top: 20px; text-align: center;">
195
- <button id="downloadBtn" class="download-btn">Télécharger le Dataset</button>
196
  </div>
197
 
198
- <h3 style="margin-top: 30px;">Aperçu des Questions</h3>
199
  <div id="questionsPreview">
200
  <table>
201
  <thead>
@@ -204,6 +362,8 @@
204
  <th>Type</th>
205
  <th>Difficulté</th>
206
  <th>But d'Entraînement</th>
 
 
207
  </tr>
208
  </thead>
209
  <tbody id="questionsTableBody">
@@ -214,105 +374,144 @@
214
  </div>
215
 
216
  <script>
217
- const apiBaseUrl = window.location.origin; // Use the same origin for API calls
218
  let generatedDataset = null;
219
  let downloadUrl = '';
220
  let statusCheckInterval = null;
 
221
 
 
222
  document.getElementById('generateBtn').addEventListener('click', startGeneration);
223
  document.getElementById('downloadBtn').addEventListener('click', downloadDataset);
 
 
 
224
 
225
- // Check for ongoing generation when the page loads
226
- window.addEventListener('load', checkOngoingGeneration);
227
-
228
- function checkOngoingGeneration() {
229
- fetch(`${apiBaseUrl}/generation-status`)
230
- .then(response => response.json())
231
- .then(status => {
232
- if (status.is_running || status.completed) {
233
- setupStatusMonitoring();
234
- updateStatusDisplay(status);
235
- }
236
-
237
- if (status.completed && status.result_file) {
238
- downloadUrl = `/download/${status.result_file}`;
239
- loadResults();
240
- }
241
- })
242
- .catch(error => {
243
- console.error('Error checking generation status:', error);
244
- });
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
245
  }
246
 
247
- function startGeneration() {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
248
  const generateBtn = document.getElementById('generateBtn');
249
  const statusContainer = document.getElementById('statusContainer');
250
  const resultsContainer = document.getElementById('resultsContainer');
251
 
252
- // Disable button and show status
253
  generateBtn.disabled = true;
254
  statusContainer.style.display = 'block';
255
  resultsContainer.style.display = 'none';
256
 
257
- // Call the API endpoint to start generation
258
- fetch(`${apiBaseUrl}/generate-questions`)
259
- .then(response => {
260
- if (!response.ok) {
261
- throw new Error(`HTTP error! Status: ${response.status}`);
262
- }
263
- return response.json();
264
- })
265
- .then(data => {
266
- console.log('Generation started:', data);
267
-
268
- // Setup status monitoring
269
- setupStatusMonitoring();
270
- })
271
- .catch(error => {
272
- console.error('Error starting generation:', error);
273
- document.getElementById('statusText').textContent = `Erreur: ${error.message}`;
274
- generateBtn.disabled = false;
275
- });
 
276
  }
277
 
278
  function setupStatusMonitoring() {
279
- // Clear any existing interval
280
  if (statusCheckInterval) {
281
  clearInterval(statusCheckInterval);
282
  }
283
 
284
- // Show the status container
285
  document.getElementById('statusContainer').style.display = 'block';
 
286
 
287
- // Start checking status regularly
288
- statusCheckInterval = setInterval(checkGenerationStatus, 5000);
289
-
290
- // Do an immediate check
291
  checkGenerationStatus();
292
  }
293
 
294
- function checkGenerationStatus() {
295
- fetch(`${apiBaseUrl}/generation-status`)
296
- .then(response => response.json())
297
- .then(status => {
298
- updateStatusDisplay(status);
299
-
300
- // If generation is completed, stop checking and load results
301
- if (status.completed) {
302
- clearInterval(statusCheckInterval);
303
- downloadUrl = `/download/${status.result_file}`;
304
- loadResults();
305
- }
306
-
307
- // If there was an error, stop checking
308
- if (status.error) {
309
- clearInterval(statusCheckInterval);
310
- document.getElementById('generateBtn').disabled = false;
311
- }
312
- })
313
- .catch(error => {
314
- console.error('Error checking status:', error);
315
- });
 
316
  }
317
 
318
  function updateStatusDisplay(status) {
@@ -321,54 +520,83 @@
321
  const processedChunks = document.getElementById('processedChunks');
322
  const totalChunks = document.getElementById('totalChunks');
323
  const questionsGenerated = document.getElementById('questionsGenerated');
 
 
 
 
324
 
325
- // Update text and counts
326
  if (status.error) {
327
- statusText.textContent = `Erreur: ${status.error}`;
328
  progressBar.classList.add('error');
329
  } else if (status.completed) {
330
- statusText.textContent = 'Génération terminée avec succès !';
331
  progressBar.classList.add('complete');
332
  } else if (status.is_running) {
333
- statusText.textContent = 'Génération des questions en cours...';
334
  }
335
 
336
- // Update progress data
337
- processedChunks.textContent = status.processed_chunks;
338
- totalChunks.textContent = status.total_chunks;
339
- questionsGenerated.textContent = status.questions_generated;
 
 
340
 
341
- // Calculate and update progress percentage
342
  if (status.total_chunks > 0) {
343
  const percentage = Math.round((status.processed_chunks / status.total_chunks) * 100);
344
  progressBar.style.width = `${percentage}%`;
345
  progressBar.textContent = `${percentage}%`;
346
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
347
  }
348
 
349
- function loadResults() {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
350
  if (!downloadUrl) return;
351
 
352
- // Fetch the generated dataset
353
- fetch(`${apiBaseUrl}${downloadUrl}`)
354
- .then(response => {
355
- if (!response.ok) {
356
- throw new Error(`HTTP error! Status: ${response.status}`);
357
- }
358
- return response.json();
359
- })
360
- .then(dataset => {
361
- generatedDataset = dataset;
362
- displayResults(dataset);
363
-
364
- // Update status
365
- const statusText = document.getElementById('statusText');
366
- statusText.textContent = 'Génération terminée avec succès !';
367
- document.getElementById('generateBtn').disabled = false;
368
- })
369
- .catch(error => {
370
- console.error('Error loading results:', error);
371
- });
372
  }
373
 
374
  function displayResults(dataset) {
@@ -376,56 +604,57 @@
376
  const statsContainer = document.getElementById('statsContainer');
377
  const questionsTableBody = document.getElementById('questionsTableBody');
378
 
379
- // Display dataset info stats
380
  statsContainer.innerHTML = '';
381
 
382
- // Total questions
383
  addStatCard(statsContainer, dataset.dataset_info.total_questions, 'Questions Totales');
 
 
384
 
385
- // Count by type
386
  const typeCount = countByProperty(dataset.questions, 'type');
387
  for (const [type, count] of Object.entries(typeCount)) {
388
- addStatCard(statsContainer, count, `Questions ${capitalizeFirstLetter(type)}`);
389
  }
390
 
391
- // Count by difficulty
392
  const difficultyCount = countByProperty(dataset.questions, 'difficulty');
393
  for (const [difficulty, count] of Object.entries(difficultyCount)) {
394
- addStatCard(statsContainer, count, `Niveau ${capitalizeFirstLetter(difficulty)}`);
 
 
 
 
 
 
395
  }
396
 
397
- // Display questions preview
398
  questionsTableBody.innerHTML = '';
399
- dataset.questions.forEach(question => {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
400
  const row = document.createElement('tr');
401
-
402
- const questionCell = document.createElement('td');
403
- questionCell.textContent = question.question;
404
-
405
- const typeCell = document.createElement('td');
406
- const typeBadge = document.createElement('span');
407
- typeBadge.className = `badge badge-${question.type}`;
408
- typeBadge.textContent = question.type;
409
- typeCell.appendChild(typeBadge);
410
-
411
- const difficultyCell = document.createElement('td');
412
- const difficultyBadge = document.createElement('span');
413
- difficultyBadge.className = `badge badge-${question.difficulty}`;
414
- difficultyBadge.textContent = question.difficulty;
415
- difficultyCell.appendChild(difficultyBadge);
416
-
417
- const purposeCell = document.createElement('td');
418
- purposeCell.textContent = question.training_purpose;
419
-
420
- row.appendChild(questionCell);
421
- row.appendChild(typeCell);
422
- row.appendChild(difficultyCell);
423
- row.appendChild(purposeCell);
424
-
425
  questionsTableBody.appendChild(row);
426
- });
427
 
428
- // Show the results container
429
  resultsContainer.style.display = 'block';
430
  }
431
 
@@ -433,16 +662,11 @@
433
  const card = document.createElement('div');
434
  card.className = 'stat-card';
435
 
436
- const valueElement = document.createElement('div');
437
- valueElement.className = 'stat-value';
438
- valueElement.textContent = value;
 
439
 
440
- const labelElement = document.createElement('div');
441
- labelElement.className = 'stat-label';
442
- labelElement.textContent = label;
443
-
444
- card.appendChild(valueElement);
445
- card.appendChild(labelElement);
446
  container.appendChild(card);
447
  }
448
 
@@ -461,9 +685,60 @@
461
 
462
  function downloadDataset() {
463
  if (downloadUrl) {
464
- window.location.href = `${apiBaseUrl}${downloadUrl}`;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
465
  }
466
  }
 
 
 
 
 
 
 
 
 
 
 
 
467
  </script>
468
  </body>
469
  </html>
 
3
  <head>
4
  <meta charset="UTF-8">
5
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
6
+ <title>Générateur de Questions sur les Vaccins - Amélioré</title>
7
  <style>
8
  body {
9
  font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
10
  line-height: 1.6;
11
  color: #333;
12
+ max-width: 1200px;
13
  margin: 0 auto;
14
  padding: 20px;
15
+ background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
16
+ min-height: 100vh;
17
  }
18
  h1 {
19
+ color: white;
20
  text-align: center;
21
+ font-size: 2.5em;
22
+ margin-bottom: 30px;
23
+ text-shadow: 2px 2px 4px rgba(0,0,0,0.3);
24
  }
25
  .container {
26
+ background: rgba(255, 255, 255, 0.95);
27
+ backdrop-filter: blur(10px);
28
+ border-radius: 15px;
29
+ box-shadow: 0 8px 32px rgba(0, 0, 0, 0.1);
30
+ padding: 25px;
31
  margin-bottom: 20px;
32
+ border: 1px solid rgba(255, 255, 255, 0.2);
33
  }
34
  button {
35
+ background: linear-gradient(45deg, #3498db, #2980b9);
36
  color: white;
37
  border: none;
38
+ padding: 12px 24px;
39
+ border-radius: 8px;
40
  cursor: pointer;
41
  font-size: 16px;
42
+ font-weight: 600;
43
+ transition: all 0.3s ease;
44
+ box-shadow: 0 4px 15px rgba(52, 152, 219, 0.3);
45
  }
46
  button:hover {
47
+ transform: translateY(-2px);
48
+ box-shadow: 0 6px 20px rgba(52, 152, 219, 0.4);
49
  }
50
  button:disabled {
51
+ background: #95a5a6;
52
  cursor: not-allowed;
53
+ transform: none;
54
+ box-shadow: none;
55
  }
56
  .download-btn {
57
+ background: linear-gradient(45deg, #27ae60, #219955);
58
+ box-shadow: 0 4px 15px rgba(39, 174, 96, 0.3);
59
  }
60
  .download-btn:hover {
61
+ box-shadow: 0 6px 20px rgba(39, 174, 96, 0.4);
62
  }
63
+ .warning-btn {
64
+ background: linear-gradient(45deg, #f39c12, #e67e22);
65
+ box-shadow: 0 4px 15px rgba(243, 156, 18, 0.3);
66
+ }
67
+ .danger-btn {
68
+ background: linear-gradient(45deg, #e74c3c, #c0392b);
69
+ box-shadow: 0 4px 15px rgba(231, 76, 60, 0.3);
70
+ }
71
+ .small-btn {
72
+ padding: 8px 16px;
73
+ font-size: 14px;
74
+ margin: 5px;
75
+ }
76
+
77
  #statusContainer {
78
  margin-top: 20px;
79
+ padding: 20px;
80
+ border-left: 5px solid #3498db;
81
+ background: linear-gradient(135deg, #e8f4fc 0%, #f0f8ff 100%);
82
+ border-radius: 10px;
83
+ }
84
+
85
+ .api-status {
86
+ background: linear-gradient(135deg, #f8f9fa 0%, #e9ecef 100%);
87
  padding: 15px;
88
+ border-radius: 10px;
89
+ margin-bottom: 15px;
90
+ border-left: 4px solid #28a745;
91
  }
92
+ .api-status.error {
93
+ border-left-color: #dc3545;
94
+ background: linear-gradient(135deg, #f8d7da 0%, #f5c6cb 100%);
95
+ }
96
+
97
  .loader {
98
  display: inline-block;
99
  width: 20px;
100
  height: 20px;
101
+ border: 3px solid rgba(255, 255, 255, 0.3);
102
  border-radius: 50%;
103
  border-top-color: #3498db;
104
  animation: spin 1s ease infinite;
 
108
  @keyframes spin {
109
  to { transform: rotate(360deg); }
110
  }
111
+
112
  .stats-container {
113
  display: grid;
114
  grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
 
116
  margin-top: 20px;
117
  }
118
  .stat-card {
119
+ background: linear-gradient(135deg, #ffffff 0%, #f8f9fa 100%);
120
+ padding: 20px;
121
+ border-radius: 12px;
122
+ box-shadow: 0 4px 15px rgba(0, 0, 0, 0.1);
123
  text-align: center;
124
+ transition: transform 0.3s ease;
125
+ }
126
+ .stat-card:hover {
127
+ transform: translateY(-5px);
128
  }
129
  .stat-value {
130
+ font-size: 28px;
131
  font-weight: bold;
132
  color: #2c3e50;
133
  margin: 10px 0;
 
135
  .stat-label {
136
  color: #7f8c8d;
137
  font-size: 14px;
138
+ font-weight: 500;
139
+ }
140
+
141
+ .progress-container {
142
+ margin-top: 15px;
143
+ background-color: #ecf0f1;
144
+ border-radius: 10px;
145
+ height: 25px;
146
+ overflow: hidden;
147
+ position: relative;
148
+ }
149
+ .progress-bar {
150
+ height: 100%;
151
+ background: linear-gradient(45deg, #3498db, #2980b9);
152
+ transition: width 0.3s ease;
153
+ display: flex;
154
+ align-items: center;
155
+ justify-content: center;
156
+ color: white;
157
+ font-size: 12px;
158
+ font-weight: bold;
159
+ position: relative;
160
+ }
161
+ .progress-bar.complete {
162
+ background: linear-gradient(45deg, #27ae60, #219955);
163
+ }
164
+ .progress-bar.error {
165
+ background: linear-gradient(45deg, #e74c3c, #c0392b);
166
+ }
167
+
168
+ .time-estimate {
169
+ background: rgba(52, 152, 219, 0.1);
170
+ padding: 10px;
171
+ border-radius: 8px;
172
+ margin-top: 10px;
173
+ font-size: 14px;
174
+ color: #2980b9;
175
+ }
176
+
177
+ .failed-chunks {
178
+ background: rgba(231, 76, 60, 0.1);
179
+ border: 1px solid rgba(231, 76, 60, 0.2);
180
+ border-radius: 8px;
181
+ padding: 15px;
182
+ margin-top: 15px;
183
+ }
184
+ .failed-chunks h4 {
185
+ color: #e74c3c;
186
+ margin-top: 0;
187
+ }
188
+ .failed-chunk-item {
189
+ background: white;
190
+ padding: 10px;
191
+ margin: 5px 0;
192
+ border-radius: 5px;
193
+ border-left: 3px solid #e74c3c;
194
  }
195
+
196
  #questionsPreview {
197
  margin-top: 20px;
198
+ max-height: 500px;
199
  overflow-y: auto;
200
+ background: rgba(255, 255, 255, 0.9);
201
+ border-radius: 12px;
202
+ box-shadow: 0 4px 15px rgba(0, 0, 0, 0.1);
203
  }
204
  table {
205
  width: 100%;
206
  border-collapse: collapse;
207
  }
208
  th, td {
209
+ padding: 15px;
210
  text-align: left;
211
+ border-bottom: 1px solid #e0e6ed;
212
  }
213
  th {
214
+ background: linear-gradient(135deg, #f8f9fa 0%, #e9ecef 100%);
215
  font-weight: 600;
216
+ color: #495057;
217
+ position: sticky;
218
+ top: 0;
219
  }
220
  tr:hover {
221
+ background-color: rgba(52, 152, 219, 0.05);
222
  }
223
+
224
  .badge {
225
  display: inline-block;
226
+ padding: 6px 12px;
227
+ border-radius: 20px;
228
+ font-size: 11px;
229
  font-weight: 600;
230
  text-transform: uppercase;
231
  color: white;
232
+ letter-spacing: 0.5px;
233
+ }
234
+ .badge-easy { background: linear-gradient(45deg, #27ae60, #2ecc71); }
235
+ .badge-medium { background: linear-gradient(45deg, #f39c12, #e67e22); }
236
+ .badge-hard { background: linear-gradient(45deg, #e74c3c, #c0392b); }
237
+ .badge-factual { background: linear-gradient(45deg, #3498db, #2980b9); }
238
+ .badge-conceptual { background: linear-gradient(45deg, #9b59b6, #8e44ad); }
239
+ .badge-applied { background: linear-gradient(45deg, #e67e22, #d35400); }
240
 
241
+ .control-panel {
 
 
 
 
 
 
 
 
 
 
242
  display: flex;
243
+ flex-wrap: wrap;
244
+ gap: 10px;
245
+ margin-bottom: 20px;
246
  align-items: center;
 
 
 
 
247
  }
248
+
249
+ .alert {
250
+ padding: 15px;
251
+ border-radius: 8px;
252
+ margin: 10px 0;
253
  }
254
+ .alert-success {
255
+ background-color: #d4edda;
256
+ border: 1px solid #c3e6cb;
257
+ color: #155724;
258
+ }
259
+ .alert-warning {
260
+ background-color: #fff3cd;
261
+ border: 1px solid #ffeaa7;
262
+ color: #856404;
263
+ }
264
+ .alert-error {
265
+ background-color: #f8d7da;
266
+ border: 1px solid #f5c6cb;
267
+ color: #721c24;
268
+ }
269
+
270
+ @media (max-width: 768px) {
271
+ body {
272
+ padding: 10px;
273
+ }
274
+ .stats-container {
275
+ grid-template-columns: repeat(auto-fill, minmax(150px, 1fr));
276
+ gap: 10px;
277
+ }
278
+ .control-panel {
279
+ flex-direction: column;
280
+ align-items: stretch;
281
+ }
282
+ button {
283
+ width: 100%;
284
+ margin: 5px 0;
285
+ }
286
  }
287
  </style>
288
  </head>
289
  <body>
290
+ <h1>🧬 Générateur de Questions sur les Vaccins</h1>
291
 
292
+ <!-- API Status Container -->
293
  <div class="container">
294
+ <h3>🔑 État des Clés API</h3>
295
+ <div id="apiStatusContainer">
296
+ <div class="api-status">
297
+ <span>Vérification des clés API...</span>
298
+ </div>
299
+ </div>
300
+ <button id="checkApiBtn" class="small-btn">Vérifier les Clés</button>
301
+ </div>
302
+
303
+ <!-- Generation Control Container -->
304
+ <div class="container">
305
+ <h2>🚀 Génération de Questions</h2>
306
+ <p>Générez des questions à partir du guide de vaccination avec gestion multi-clés et suivi en temps réel.</p>
307
+
308
+ <div class="control-panel">
309
+ <button id="generateBtn">Générer des Questions</button>
310
+ <button id="downloadProgressBtn" class="download-btn small-btn" style="display: none;">Télécharger Progrès</button>
311
+ <button id="retryFailedBtn" class="warning-btn small-btn" style="display: none;">Voir Échecs</button>
312
+ </div>
313
 
314
  <div id="statusContainer" style="display: none;">
315
  <div id="statusHeader">
 
321
  <div id="progressBar" class="progress-bar" style="width: 0%">0%</div>
322
  </div>
323
 
324
+ <div id="statusDetails" style="margin-top: 15px;">
325
+ <div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 15px;">
326
+ <div><strong>Chunks traités:</strong> <span id="processedChunks">0</span>/<span id="totalChunks">0</span></div>
327
+ <div><strong>Questions générées:</strong> <span id="questionsGenerated">0</span></div>
328
+ <div><strong>Clé API actuelle:</strong> <span id="currentApiKey">-</span></div>
329
+ <div><strong>Échecs:</strong> <span id="failedChunks">0</span></div>
330
+ </div>
331
+ </div>
332
+
333
+ <div id="timeEstimate" class="time-estimate" style="display: none;">
334
+ <strong>⏱️ Temps estimé restant:</strong> <span id="estimatedTime">Calcul en cours...</span>
335
+ </div>
336
+
337
+ <div id="failedChunksContainer" class="failed-chunks" style="display: none;">
338
+ <h4>❌ Chunks Échoués</h4>
339
+ <div id="failedChunksList"></div>
340
  </div>
341
  </div>
342
  </div>
343
 
344
+ <!-- Results Container -->
345
  <div class="container" id="resultsContainer" style="display: none;">
346
+ <h2>📊 Résultats de Génération</h2>
347
 
348
  <div class="stats-container" id="statsContainer">
349
  <!-- Stats will be populated here -->
350
  </div>
351
 
352
+ <div style="margin-top: 25px; text-align: center;">
353
+ <button id="downloadBtn" class="download-btn">📥 Télécharger le Dataset Complet</button>
354
  </div>
355
 
356
+ <h3 style="margin-top: 30px;">👀 Aperçu des Questions</h3>
357
  <div id="questionsPreview">
358
  <table>
359
  <thead>
 
362
  <th>Type</th>
363
  <th>Difficulté</th>
364
  <th>But d'Entraînement</th>
365
+ <th>Chunk ID</th>
366
+ <th>Clé API</th>
367
  </tr>
368
  </thead>
369
  <tbody id="questionsTableBody">
 
374
  </div>
375
 
376
  <script>
377
+ const apiBaseUrl = window.location.origin;
378
  let generatedDataset = null;
379
  let downloadUrl = '';
380
  let statusCheckInterval = null;
381
+ let currentStatus = null;
382
 
383
+ // Event listeners
384
  document.getElementById('generateBtn').addEventListener('click', startGeneration);
385
  document.getElementById('downloadBtn').addEventListener('click', downloadDataset);
386
+ document.getElementById('downloadProgressBtn').addEventListener('click', downloadProgress);
387
+ document.getElementById('checkApiBtn').addEventListener('click', checkApiKeys);
388
+ document.getElementById('retryFailedBtn').addEventListener('click', showFailedChunks);
389
 
390
+ // Check for ongoing generation and API status when the page loads
391
+ window.addEventListener('load', () => {
392
+ checkApiKeys();
393
+ checkOngoingGeneration();
394
+ });
395
+
396
+ async function checkApiKeys() {
397
+ try {
398
+ const response = await fetch(`${apiBaseUrl}/api-keys-status`);
399
+ const status = await response.json();
400
+
401
+ const container = document.getElementById('apiStatusContainer');
402
+
403
+ if (status.status === 'success') {
404
+ container.innerHTML = `
405
+ <div class="api-status">
406
+ <strong>✅ ${status.total_keys} clés API configurées</strong><br>
407
+ <small>Clé actuelle: Index ${status.current_key_index} | Rotation automatique activée</small>
408
+ </div>
409
+ `;
410
+ } else {
411
+ container.innerHTML = `
412
+ <div class="api-status error">
413
+ <strong>❌ Erreur de configuration API</strong><br>
414
+ <small>${status.message}</small>
415
+ </div>
416
+ `;
417
+ }
418
+ } catch (error) {
419
+ console.error('Error checking API keys:', error);
420
+ document.getElementById('apiStatusContainer').innerHTML = `
421
+ <div class="api-status error">
422
+ <strong>❌ Impossible de vérifier les clés API</strong><br>
423
+ <small>${error.message}</small>
424
+ </div>
425
+ `;
426
+ }
427
  }
428
 
429
+ async function checkOngoingGeneration() {
430
+ try {
431
+ const response = await fetch(`${apiBaseUrl}/generation-status`);
432
+ const status = await response.json();
433
+ currentStatus = status;
434
+
435
+ if (status.is_running || status.completed) {
436
+ setupStatusMonitoring();
437
+ updateStatusDisplay(status);
438
+ }
439
+
440
+ if (status.completed && status.result_file) {
441
+ downloadUrl = `/download/${status.result_file}`;
442
+ await loadResults();
443
+ }
444
+ } catch (error) {
445
+ console.error('Error checking generation status:', error);
446
+ }
447
+ }
448
+
449
+ async function startGeneration() {
450
  const generateBtn = document.getElementById('generateBtn');
451
  const statusContainer = document.getElementById('statusContainer');
452
  const resultsContainer = document.getElementById('resultsContainer');
453
 
 
454
  generateBtn.disabled = true;
455
  statusContainer.style.display = 'block';
456
  resultsContainer.style.display = 'none';
457
 
458
+ try {
459
+ const response = await fetch(`${apiBaseUrl}/generate-questions`);
460
+
461
+ if (!response.ok) {
462
+ throw new Error(`HTTP error! Status: ${response.status}`);
463
+ }
464
+
465
+ const data = await response.json();
466
+ console.log('Generation started:', data);
467
+
468
+ // Show success alert
469
+ showAlert('Génération démarrée avec succès!', 'success');
470
+
471
+ setupStatusMonitoring();
472
+ } catch (error) {
473
+ console.error('Error starting generation:', error);
474
+ showAlert(`Erreur lors du démarrage: ${error.message}`, 'error');
475
+ document.getElementById('statusText').textContent = `Erreur: ${error.message}`;
476
+ generateBtn.disabled = false;
477
+ }
478
  }
479
 
480
  function setupStatusMonitoring() {
 
481
  if (statusCheckInterval) {
482
  clearInterval(statusCheckInterval);
483
  }
484
 
 
485
  document.getElementById('statusContainer').style.display = 'block';
486
+ document.getElementById('downloadProgressBtn').style.display = 'inline-block';
487
 
488
+ statusCheckInterval = setInterval(checkGenerationStatus, 3000);
 
 
 
489
  checkGenerationStatus();
490
  }
491
 
492
+ async function checkGenerationStatus() {
493
+ try {
494
+ const response = await fetch(`${apiBaseUrl}/generation-status`);
495
+ const status = await response.json();
496
+ currentStatus = status;
497
+
498
+ updateStatusDisplay(status);
499
+
500
+ if (status.completed) {
501
+ clearInterval(statusCheckInterval);
502
+ downloadUrl = `/download/${status.result_file}`;
503
+ await loadResults();
504
+ showAlert('Génération terminée avec succès!', 'success');
505
+ }
506
+
507
+ if (status.error) {
508
+ clearInterval(statusCheckInterval);
509
+ document.getElementById('generateBtn').disabled = false;
510
+ showAlert(`Erreur durant la génération: ${status.error}`, 'error');
511
+ }
512
+ } catch (error) {
513
+ console.error('Error checking status:', error);
514
+ }
515
  }
516
 
517
  function updateStatusDisplay(status) {
 
520
  const processedChunks = document.getElementById('processedChunks');
521
  const totalChunks = document.getElementById('totalChunks');
522
  const questionsGenerated = document.getElementById('questionsGenerated');
523
+ const currentApiKey = document.getElementById('currentApiKey');
524
+ const failedChunks = document.getElementById('failedChunks');
525
+ const timeEstimate = document.getElementById('timeEstimate');
526
+ const estimatedTime = document.getElementById('estimatedTime');
527
 
528
+ // Update status text
529
  if (status.error) {
530
+ statusText.textContent = `❌ Erreur: ${status.error}`;
531
  progressBar.classList.add('error');
532
  } else if (status.completed) {
533
+ statusText.textContent = 'Génération terminée avec succès !';
534
  progressBar.classList.add('complete');
535
  } else if (status.is_running) {
536
+ statusText.textContent = 'Génération des questions en cours...';
537
  }
538
 
539
+ // Update counts and progress
540
+ processedChunks.textContent = status.processed_chunks || 0;
541
+ totalChunks.textContent = status.total_chunks || 0;
542
+ questionsGenerated.textContent = status.questions_generated || 0;
543
+ currentApiKey.textContent = status.current_api_key_index || 0;
544
+ failedChunks.textContent = (status.failed_chunks || []).length;
545
 
546
+ // Update progress bar
547
  if (status.total_chunks > 0) {
548
  const percentage = Math.round((status.processed_chunks / status.total_chunks) * 100);
549
  progressBar.style.width = `${percentage}%`;
550
  progressBar.textContent = `${percentage}%`;
551
  }
552
+
553
+ // Show time estimate
554
+ if (status.estimated_remaining_minutes) {
555
+ timeEstimate.style.display = 'block';
556
+ const minutes = Math.round(status.estimated_remaining_minutes);
557
+ estimatedTime.textContent = `${minutes} minutes`;
558
+ }
559
+
560
+ // Show failed chunks
561
+ if (status.failed_chunks && status.failed_chunks.length > 0) {
562
+ document.getElementById('retryFailedBtn').style.display = 'inline-block';
563
+ updateFailedChunksDisplay(status.failed_chunks);
564
+ }
565
  }
566
 
567
+ function updateFailedChunksDisplay(failedChunks) {
568
+ const container = document.getElementById('failedChunksContainer');
569
+ const list = document.getElementById('failedChunksList');
570
+
571
+ if (failedChunks.length > 0) {
572
+ container.style.display = 'block';
573
+ list.innerHTML = failedChunks.map(chunk => `
574
+ <div class="failed-chunk-item">
575
+ <strong>Chunk ${chunk.chunk_id}:</strong> ${chunk.error}<br>
576
+ <small>Tentatives: ${chunk.attempts}</small>
577
+ </div>
578
+ `).join('');
579
+ }
580
+ }
581
+
582
+ async function loadResults() {
583
  if (!downloadUrl) return;
584
 
585
+ try {
586
+ const response = await fetch(`${apiBaseUrl}${downloadUrl}`);
587
+ if (!response.ok) {
588
+ throw new Error(`HTTP error! Status: ${response.status}`);
589
+ }
590
+
591
+ const dataset = await response.json();
592
+ generatedDataset = dataset;
593
+ displayResults(dataset);
594
+
595
+ document.getElementById('generateBtn').disabled = false;
596
+ } catch (error) {
597
+ console.error('Error loading results:', error);
598
+ showAlert(`Erreur lors du chargement des résultats: ${error.message}`, 'error');
599
+ }
 
 
 
 
 
600
  }
601
 
602
  function displayResults(dataset) {
 
604
  const statsContainer = document.getElementById('statsContainer');
605
  const questionsTableBody = document.getElementById('questionsTableBody');
606
 
607
+ // Clear and populate stats
608
  statsContainer.innerHTML = '';
609
 
610
+ // Main stats
611
  addStatCard(statsContainer, dataset.dataset_info.total_questions, 'Questions Totales');
612
+ addStatCard(statsContainer, dataset.dataset_info.successful_chunks || 0, 'Chunks Réussis');
613
+ addStatCard(statsContainer, dataset.dataset_info.failed_chunks || 0, 'Chunks Échoués');
614
 
615
+ // Type distribution
616
  const typeCount = countByProperty(dataset.questions, 'type');
617
  for (const [type, count] of Object.entries(typeCount)) {
618
+ addStatCard(statsContainer, count, `${capitalizeFirstLetter(type)}`);
619
  }
620
 
621
+ // Difficulty distribution
622
  const difficultyCount = countByProperty(dataset.questions, 'difficulty');
623
  for (const [difficulty, count] of Object.entries(difficultyCount)) {
624
+ addStatCard(statsContainer, count, `${capitalizeFirstLetter(difficulty)}`);
625
+ }
626
+
627
+ // Success rate
628
+ if (dataset.dataset_info.total_chunks_processed) {
629
+ const successRate = Math.round(((dataset.dataset_info.successful_chunks || 0) / dataset.dataset_info.total_chunks_processed) * 100);
630
+ addStatCard(statsContainer, `${successRate}%`, 'Taux de Réussite');
631
  }
632
 
633
+ // Populate questions table
634
  questionsTableBody.innerHTML = '';
635
+ dataset.questions.forEach((question, index) => {
636
+ if (index < 50) { // Limit display for performance
637
+ const row = document.createElement('tr');
638
+
639
+ row.innerHTML = `
640
+ <td>${question.question}</td>
641
+ <td><span class="badge badge-${question.type}">${question.type}</span></td>
642
+ <td><span class="badge badge-${question.difficulty}">${question.difficulty}</span></td>
643
+ <td>${question.training_purpose}</td>
644
+ <td>${question.chunk_id}</td>
645
+ <td>${question.api_key_used || 'N/A'}</td>
646
+ `;
647
+
648
+ questionsTableBody.appendChild(row);
649
+ }
650
+ });
651
+
652
+ if (dataset.questions.length > 50) {
653
  const row = document.createElement('tr');
654
+ row.innerHTML = `<td colspan="6" style="text-align: center; font-style: italic; color: #666;">... et ${dataset.questions.length - 50} autres questions</td>`;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
655
  questionsTableBody.appendChild(row);
656
+ }
657
 
 
658
  resultsContainer.style.display = 'block';
659
  }
660
 
 
662
  const card = document.createElement('div');
663
  card.className = 'stat-card';
664
 
665
+ card.innerHTML = `
666
+ <div class="stat-value">${value}</div>
667
+ <div class="stat-label">${label}</div>
668
+ `;
669
 
 
 
 
 
 
 
670
  container.appendChild(card);
671
  }
672
 
 
685
 
686
  function downloadDataset() {
687
  if (downloadUrl) {
688
+ window.open(`${apiBaseUrl}${downloadUrl}`, '_blank');
689
+ }
690
+ }
691
+
692
+ async function downloadProgress() {
693
+ try {
694
+ const response = await fetch(`${apiBaseUrl}/download-progress`);
695
+ if (response.ok) {
696
+ const blob = await response.blob();
697
+ const url = window.URL.createObjectURL(blob);
698
+ const a = document.createElement('a');
699
+ a.href = url;
700
+ a.download = `progress_${new Date().getTime()}.json`;
701
+ document.body.appendChild(a);
702
+ a.click();
703
+ window.URL.revokeObjectURL(url);
704
+ document.body.removeChild(a);
705
+ showAlert('Fichier de progrès téléchargé!', 'success');
706
+ } else {
707
+ throw new Error('Pas de fichier de progrès disponible');
708
+ }
709
+ } catch (error) {
710
+ showAlert(`Erreur de téléchargement: ${error.message}`, 'error');
711
+ }
712
+ }
713
+
714
+ async function showFailedChunks() {
715
+ try {
716
+ const response = await fetch(`${apiBaseUrl}/retry-failed`);
717
+ const data = await response.json();
718
+
719
+ if (data.failed_chunks && data.failed_chunks.length > 0) {
720
+ updateFailedChunksDisplay(data.failed_chunks);
721
+ document.getElementById('failedChunksContainer').style.display = 'block';
722
+ showAlert(`${data.failed_chunks.length} chunks ont échoué. Voir les détails ci-dessous.`, 'warning');
723
+ } else {
724
+ showAlert('Aucun chunk échoué trouvé!', 'success');
725
+ }
726
+ } catch (error) {
727
+ showAlert(`Erreur: ${error.message}`, 'error');
728
  }
729
  }
730
+
731
+ function showAlert(message, type) {
732
+ const alertDiv = document.createElement('div');
733
+ alertDiv.className = `alert alert-${type}`;
734
+ alertDiv.textContent = message;
735
+
736
+ document.body.insertBefore(alertDiv, document.body.firstChild);
737
+
738
+ setTimeout(() => {
739
+ alertDiv.remove();
740
+ }, 5000);
741
+ }
742
  </script>
743
  </body>
744
  </html>