Update app.py

app.py

@@ -1,15 +1,49 @@
 import os
 import time
+import signal
 import gradio as gr
 import requests
 import pandas as pd
 from langchain_core.messages import HumanMessage
-from agent import build_graph, extract_final_answer
+from agent import build_graph, extract_final_answer, sort_questions_by_difficulty, classify_difficulty, QUESTION_TIMEOUT_SECONDS
 
 # --- Constants ---
 DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 
 
+# ============================================================
+# TIMEOUT HANDLER
+# ============================================================
+
+class QuestionTimeoutError(Exception):
+    pass
+
+
+def timeout_handler(signum, frame):
+    raise QuestionTimeoutError("Question timed out")
+
+
+def run_with_timeout(fn, timeout_seconds):
+    """Run fn() with a hard timeout. Returns (result, timed_out)."""
+    # Use SIGALRM on Unix systems (works on Linux/HF Spaces)
+    try:
+        signal.signal(signal.SIGALRM, timeout_handler)
+        signal.alarm(timeout_seconds)
+        result = fn()
+        signal.alarm(0)  # cancel alarm
+        return result, False
+    except QuestionTimeoutError:
+        signal.alarm(0)
+        return None, True
+    except Exception as e:
+        signal.alarm(0)
+        raise e
+
+
+# ============================================================
+# AGENT CLASS
+# ============================================================
+
 class BasicAgent:
     """A LangGraph agent with multi-model fallback."""
 
@@ -22,11 +56,8 @@ class BasicAgent:
 
         inputs = {"messages": [HumanMessage(content=question)]}
         result = self.graph.invoke(inputs)
-
-        # Use the robust extractor from agent.py
         answer = extract_final_answer(result)
 
-        # Guard against empty answers
         if not answer or not answer.strip():
             answer = "N/A"
 
@@ -34,13 +65,16 @@ class BasicAgent:
         return answer
 
 
+# ============================================================
+# MAIN EVALUATION FUNCTION
+# ============================================================
+
 def run_and_submit_all(profile: gr.OAuthProfile | None):
-    """Fetches questions, runs the LangGraph agent, and submits results."""
+    """Fetches questions, sorts by difficulty, runs agent with timeout, submits."""
 
-    # Check SPACE_ID early
     space_id = os.getenv("SPACE_ID")
     if not space_id:
-        return "SPACE_ID environment variable is not set. Add it in your Space settings.", None
+        return "SPACE_ID environment variable is not set.", None
 
     if profile:
         username = profile.username
@@ -72,50 +106,78 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
     if not questions_data:
         return "No questions received from the server.", None
 
+    # ✅ Sort questions: simple → medium → hard
+    original_count = len(questions_data)
+    questions_data = sort_questions_by_difficulty(questions_data)
+    print(f"\n📊 Question difficulty breakdown:")
+    for item in questions_data:
+        q = item.get("question", "")
+        d = classify_difficulty(q)
+        print(f"  [{d.upper():6}] {q[:60]}...")
+
     # Run Agent on Each Question
     results_log = []
     answers_payload = []
+    skipped = 0
+    timed_out = 0
 
     for i, item in enumerate(questions_data):
         task_id = item.get("task_id")
         question_text = item.get("question")
-        print(f"\n--- Question {i+1}/{len(questions_data)} | Task ID: {task_id} ---")
-        print(f"Question: {question_text[:100]}...")
+        difficulty = classify_difficulty(question_text)
+
+        print(f"\n--- Question {i+1}/{original_count} | [{difficulty.upper()}] | Task ID: {task_id} ---")
+        print(f"Q: {question_text[:100]}...")
+
+        start_time = time.time()
 
         try:
-            submitted_answer = agent(question_text)
-
-            # ✅ CORRECT FIELD NAME: "submitted_answer" (NOT "model_answer")
-            answers_payload.append({
-                "task_id": task_id,
-                "submitted_answer": submitted_answer
-            })
-            results_log.append({
-                "Task ID": task_id,
-                "Question": question_text,
-                "Submitted Answer": submitted_answer,
-                "Status": "OK"
-            })
+            # Run with hard timeout
+            def run_agent():
+                return agent(question_text)
+
+            submitted_answer, did_timeout = run_with_timeout(run_agent, QUESTION_TIMEOUT_SECONDS)
+
+            elapsed = time.time() - start_time
+
+            if did_timeout:
+                print(f"⏰ TIMED OUT after {QUESTION_TIMEOUT_SECONDS}s — skipping.")
+                timed_out += 1
+                status = f"TIMEOUT ({QUESTION_TIMEOUT_SECONDS}s)"
+                submitted_answer = "N/A"
+            else:
+                print(f"✅ Answered in {elapsed:.1f}s")
+                status = f"OK ({elapsed:.1f}s)"
 
         except Exception as e:
-            print(f"Error on task {task_id}: {e}")
-            # Still append a placeholder so we don't skip the task entirely
-            answers_payload.append({
-                "task_id": task_id,
-                "submitted_answer": "N/A"
-            })
-            results_log.append({
-                "Task ID": task_id,
-                "Question": question_text,
-                "Submitted Answer": f"ERROR: {e}",
-                "Status": "Error"
-            })
-
-        # Delay between questions to avoid rate limits
-        time.sleep(5)
+            elapsed = time.time() - start_time
+            print(f"❌ Error on task {task_id} after {elapsed:.1f}s: {e}")
+            submitted_answer = "N/A"
+            status = f"ERROR ({elapsed:.1f}s)"
+
+        # Always append — even skipped/timed-out questions get "N/A"
+        answers_payload.append({
+            "task_id": task_id,
+            "submitted_answer": str(submitted_answer).strip()
+        })
+        results_log.append({
+            "Task ID": task_id,
+            "Difficulty": difficulty.upper(),
+            "Question": question_text[:80],
+            "Submitted Answer": submitted_answer,
+            "Status": status
+        })
+
+        # Short delay between questions
+        delay = 3 if difficulty == "simple" else 5
+        time.sleep(delay)
 
     if not answers_payload:
-        return "Agent failed on all questions. Check logs.", pd.DataFrame(results_log)
+        return "Agent failed on all questions.", pd.DataFrame(results_log)
+
+    # Summary before submit
+    answered = sum(1 for r in results_log if r["Submitted Answer"] != "N/A")
+    print(f"\n📊 Summary: {answered}/{original_count} answered | {timed_out} timed out | {skipped} skipped")
 
     # Submit Answers
     submission_data = {
@@ -132,48 +194,59 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
         result_data = response.json()
 
         final_status = (
-            f"Submission Successful!\n"
+            f"✅ Submission Successful!\n"
             f"User: {result_data.get('username')}\n"
-            f"Overall Score: {result_data.get('score', 'N/A')}% "
-            f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
+            f"Score: {result_data.get('score', 'N/A')}%\n"
+            f"Correct: {result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')}\n"
+            f"Answered: {answered}/{original_count} | Timed out: {timed_out}\n"
             f"Message: {result_data.get('message', '')}"
         )
         print(final_status)
         return final_status, pd.DataFrame(results_log)
 
     except requests.exceptions.HTTPError as e:
-        # Show the actual API error body for easier debugging
         try:
             error_detail = e.response.json()
         except Exception:
             error_detail = e.response.text
-        msg = f"Submission Failed: {e}\nAPI Response: {error_detail}"
+        msg = f"❌ Submission Failed (HTTP {e.response.status_code}):\n{error_detail}"
         print(msg)
         return msg, pd.DataFrame(results_log)
 
     except Exception as e:
-        msg = f"Submission Failed: {e}"
+        msg = f"❌ Submission Failed: {e}"
         print(msg)
         return msg, pd.DataFrame(results_log)
 
 
-# --- Gradio UI ---
+# ============================================================
+# GRADIO UI
+# ============================================================
+
 with gr.Blocks() as demo:
-    gr.Markdown("# LangGraph Agent — GAIA Evaluation")
-    gr.Markdown("**Instructions:**")
-    gr.Markdown("1. Login with your Hugging Face account")
-    gr.Markdown("2. Click **Run Evaluation** to start")
-    gr.Markdown("3. The agent will answer all questions and submit automatically")
-    gr.Markdown("_Note: This may take several minutes due to rate-limit delays between questions._")
+    gr.Markdown("# 🤖 LangGraph GAIA Benchmark Agent")
+
+    gr.Markdown("""
+    **Strategy:**
+    - ✅ Questions sorted: Simple → Medium → Hard
+    - ⏰ Hard questions skipped if they exceed 90 seconds
+    - 🎯 Target: 30%+ success rate on simple questions first
+    - 🔄 Model stack: Qwen3 32B → Llama 3.3 70B → Llama 3.1 8B
+
+    **Instructions:**
+    1. Login with Hugging Face
+    2. Click **Run Evaluation**
+    3. Agent answers all questions and submits automatically
+    """)
 
     gr.LoginButton()
 
-    run_button = gr.Button("Run Evaluation and Submit All Answers", variant="primary")
+    run_button = gr.Button("Run Evaluation and Submit All Answers", variant="primary", size="lg")
 
     status_output = gr.Textbox(
         label="Submission Status",
-        lines=6,
-        placeholder="Status will appear here after submission..."
+        lines=8,
+        interactive=False
     )
 
     results_table = gr.DataFrame(