Update app.py

app.py

@@ -1,279 +1,365 @@
 import os
 import time
 import threading
-import gradio as gr
 import requests
 import pandas as pd
+import gradio as gr
+
 from langchain_core.messages import HumanMessage
-from agent import build_graph, extract_final_answer, sort_questions_by_difficulty, classify_difficulty, QUESTION_TIMEOUT_SECONDS
 
-# --- Constants ---
+from agent import (
+    build_graph,
+    extract_final_answer,
+    classify_question,
+)
+
+# ============================================================
+# CONFIG
+# ============================================================
+
 DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 
+FETCH_RETRIES = 3
+SUBMIT_RETRIES = 3
+
+SLEEP_SIMPLE = 0.5
+SLEEP_MEDIUM = 1
+SLEEP_HARD = 1.5
+
+TIMEOUTS = {
+    "simple": 25,
+    "medium": 45,
+    "hard": 75
+}
 
 # ============================================================
-# THREADING-BASED TIMEOUT (works in Gradio worker threads)
+# THREAD TIMEOUT
 # ============================================================
 
 def run_with_timeout(fn, timeout_seconds):
     """
-    Run fn() in a separate thread with a hard timeout.
-    Returns (result, timed_out).
-    Uses threading instead of signal — safe in non-main threads.
+    Run fn() in background thread.
+    Returns (result, timed_out)
     """
-    result_container = {"result": None, "error": None, "done": False}
+    state = {
+        "done": False,
+        "result": None,
+        "error": None
+    }
 
     def target():
         try:
-            result_container["result"] = fn()
+            state["result"] = fn()
         except Exception as e:
-            result_container["error"] = e
+            state["error"] = e
         finally:
-            result_container["done"] = True
+            state["done"] = True
 
     thread = threading.Thread(target=target, daemon=True)
     thread.start()
-    thread.join(timeout=timeout_seconds)
+    thread.join(timeout_seconds)
 
-    if not result_container["done"]:
-        # Thread is still running — treat as timeout
-        # The daemon thread will be killed when the process exits
+    if not state["done"]:
         return None, True
 
-    if result_container["error"] is not None:
-        raise result_container["error"]
+    if state["error"]:
+        raise state["error"]
 
-    return result_container["result"], False
+    return state["result"], False
 
 
 # ============================================================
-# AGENT CLASS
+# AGENT WRAPPER
 # ============================================================
 
-class BasicAgent:
-    """A LangGraph agent with multi-model fallback."""
-
+class BenchmarkAgent:
     def __init__(self):
-        print("BasicAgent (LangGraph) initialized.")
         self.graph = build_graph()
 
     def __call__(self, question: str) -> str:
-        print(f"Agent received question: {question[:80]}...")
+        inputs = {
+            "messages": [HumanMessage(content=question)]
+        }
 
-        inputs = {"messages": [HumanMessage(content=question)]}
         result = self.graph.invoke(inputs)
+
         answer = extract_final_answer(result)
 
         if not answer or not answer.strip():
-            answer = "N/A"
+            return "N/A"
 
-        print(f"Final answer: {answer}")
-        return answer
+        return answer.strip()
 
 
 # ============================================================
-# MAIN EVALUATION FUNCTION
+# HELPERS
 # ============================================================
 
-def run_and_submit_all(profile: gr.OAuthProfile | None):
-    """Fetches questions, sorts by difficulty, runs agent with timeout, submits."""
+def fetch_questions(url):
+    last_err = None
+
+    for i in range(FETCH_RETRIES):
+        try:
+            r = requests.get(url, timeout=20)
+            r.raise_for_status()
+            return r.json()
+
+        except Exception as e:
+            last_err = e
+            time.sleep(2)
+
+    raise RuntimeError(last_err)
+
+
+def submit_answers(url, payload):
+    last_err = None
+
+    for i in range(SUBMIT_RETRIES):
+        try:
+            r = requests.post(url, json=payload, timeout=60)
+            r.raise_for_status()
+            return r.json()
+
+        except Exception as e:
+            last_err = e
+            time.sleep(3)
+
+    raise RuntimeError(last_err)
+
+
+def sort_questions(questions):
+    order = {
+        "simple": 0,
+        "medium": 1,
+        "hard": 2
+    }
+
+    return sorted(
+        questions,
+        key=lambda x: order.get(
+            classify_question(x.get("question", "")),
+            9
+        )
+    )
+
+
+# ============================================================
+# MAIN RUNNER
+# ============================================================
 
+def run_and_submit_all(profile: gr.OAuthProfile | None):
     space_id = os.getenv("SPACE_ID")
+
     if not space_id:
-        return "SPACE_ID environment variable is not set.", None
+        return "SPACE_ID not set.", None
+
+    if not profile:
+        return "Please login with Hugging Face.", None
 
-    if profile:
-        username = profile.username
-        print(f"Logged in as: {username}")
-    else:
-        return "Please Login to Hugging Face with the button.", None
+    username = profile.username.strip()
 
     api_url = DEFAULT_API_URL
+
     questions_url = f"{api_url}/questions"
     submit_url = f"{api_url}/submit"
+
     agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
 
-    # Instantiate Agent
+    # --------------------------------------------------------
+    # INIT AGENT
+    # --------------------------------------------------------
+
     try:
-        agent = BasicAgent()
+        agent = BenchmarkAgent()
     except Exception as e:
-        return f"Error initializing agent: {e}", None
+        return f"Agent init failed: {e}", None
+
+    # --------------------------------------------------------
+    # FETCH QUESTIONS
+    # --------------------------------------------------------
 
-    # Fetch Questions
     try:
-        print(f"Fetching questions from {questions_url}...")
-        response = requests.get(questions_url, timeout=15)
-        response.raise_for_status()
-        questions_data = response.json()
-        print(f"Fetched {len(questions_data)} questions.")
+        questions = fetch_questions(questions_url)
     except Exception as e:
-        return f"Error fetching questions: {e}", None
+        return f"Failed to fetch questions: {e}", None
+
+    if not questions:
+        return "No questions returned.", None
+
+    questions = sort_questions(questions)
 
-    if not questions_data:
-        return "No questions received from the server.", None
+    total = len(questions)
 
-    # ✅ Sort questions: simple → medium → hard
-    original_count = len(questions_data)
-    questions_data = sort_questions_by_difficulty(questions_data)
+    # --------------------------------------------------------
+    # RUN QUESTIONS
+    # --------------------------------------------------------
 
-    print(f"\n📊 Question difficulty breakdown:")
-    for item in questions_data:
-        q = item.get("question", "")
-        d = classify_difficulty(q)
-        print(f"  [{d.upper():6}] {q[:60]}...")
+    logs = []
+    answers = []
 
-    # Run Agent on Each Question
-    results_log = []
-    answers_payload = []
-    timed_out_count = 0
+    answered = 0
+    timed_out = 0
+    errors = 0
 
-    for i, item in enumerate(questions_data):
+    for idx, item in enumerate(questions, start=1):
         task_id = item.get("task_id")
-        question_text = item.get("question")
-        difficulty = classify_difficulty(question_text)
+        question = item.get("question", "")
 
-        # Give hard questions slightly more time
-        timeout = QUESTION_TIMEOUT_SECONDS
-        if difficulty == "simple":
-            timeout = 60
-        elif difficulty == "medium":
-            timeout = 90
-        else:
-            timeout = 120  # hard questions get 2 minutes
+        difficulty = classify_question(question)
+        timeout = TIMEOUTS[difficulty]
 
-        print(f"\n--- Question {i+1}/{original_count} | [{difficulty.upper()}] | timeout={timeout}s | Task ID: {task_id} ---")
-        print(f"Q: {question_text[:100]}...")
+        start = time.time()
 
-        start_time = time.time()
-        submitted_answer = "N/A"
-        status = "SKIPPED"
+        answer = "N/A"
+        status = "UNKNOWN"
 
         try:
-            def run_agent():
-                return agent(question_text)
+            def solve():
+                return agent(question)
 
-            result, did_timeout = run_with_timeout(run_agent, timeout)
-            elapsed = time.time() - start_time
+            result, did_timeout = run_with_timeout(
+                solve,
+                timeout
+            )
+
+            elapsed = round(time.time() - start, 1)
 
             if did_timeout:
-                print(f"⏰ TIMED OUT after {elapsed:.1f}s — submitting N/A.")
-                timed_out_count += 1
-                status = f"TIMEOUT ({elapsed:.1f}s)"
-                submitted_answer = "N/A"
+                timed_out += 1
+                status = f"TIMEOUT ({elapsed}s)"
+                answer = "N/A"
+
             else:
-                submitted_answer = result if result else "N/A"
-                print(f"✅ Answered in {elapsed:.1f}s: {submitted_answer}")
-                status = f"OK ({elapsed:.1f}s)"
+                answer = result if result else "N/A"
+
+                if answer != "N/A":
+                    answered += 1
+
+                status = f"OK ({elapsed}s)"
 
         except Exception as e:
-            elapsed = time.time() - start_time
-            print(f"❌ Error on task {task_id} after {elapsed:.1f}s: {e}")
-            submitted_answer = "N/A"
-            status = f"ERROR ({elapsed:.1f}s)"
+            elapsed = round(time.time() - start, 1)
+            errors += 1
+            status = f"ERROR ({elapsed}s)"
+            answer = "N/A"
 
-        # Always append — even failed questions
-        answers_payload.append({
+        answers.append({
             "task_id": task_id,
-            "submitted_answer": str(submitted_answer).strip()
+            "submitted_answer": str(answer).strip()
         })
-        results_log.append({
+
+        logs.append({
+            "No": idx,
             "Task ID": task_id,
             "Difficulty": difficulty.upper(),
-            "Question": question_text[:80],
-            "Submitted Answer": submitted_answer,
-            "Status": status
+            "Submitted Answer": answer,
+            "Status": status,
+            "Question": question[:120]
         })
 
-        # Short delay between questions
-        delay = 2 if difficulty == "simple" else 4
-        time.sleep(delay)
+        # small pacing delay
+        if difficulty == "simple":
+            time.sleep(SLEEP_SIMPLE)
+        elif difficulty == "medium":
+            time.sleep(SLEEP_MEDIUM)
+        else:
+            time.sleep(SLEEP_HARD)
 
-    if not answers_payload:
-        return "Agent failed on all questions.", pd.DataFrame(results_log)
+    # --------------------------------------------------------
+    # SAVE LOCAL CSV
+    # --------------------------------------------------------
 
-    # Summary before submit
-    answered = sum(1 for r in results_log if r["Submitted Answer"] != "N/A")
-    print(f"\n📊 Summary: {answered}/{original_count} answered | {timed_out_count} timed out")
+    df = pd.DataFrame(logs)
+
+    try:
+        df.to_csv("last_run_results.csv", index=False)
+    except Exception:
+        pass
 
-    # Submit Answers
-    submission_data = {
-        "username": username.strip(),
+    # --------------------------------------------------------
+    # SUBMIT
+    # --------------------------------------------------------
+
+    payload = {
+        "username": username,
         "agent_code": agent_code,
-        "answers": answers_payload
+        "answers": answers
     }
 
-    print(f"\nSubmitting {len(answers_payload)} answers for user '{username}'...")
-
     try:
-        response = requests.post(submit_url, json=submission_data, timeout=60)
-        response.raise_for_status()
-        result_data = response.json()
-
-        final_status = (
-            f"✅ Submission Successful!\n"
-            f"User: {result_data.get('username')}\n"
-            f"Score: {result_data.get('score', 'N/A')}%\n"
-            f"Correct: {result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')}\n"
-            f"Answered: {answered}/{original_count} | Timed out: {timed_out_count}\n"
-            f"Message: {result_data.get('message', '')}"
+        result = submit_answers(submit_url, payload)
+
+        status = (
+            f"SUCCESS\n\n"
+            f"User: {result.get('username')}\n"
+            f"Score: {result.get('score', 'N/A')}%\n"
+            f"Correct: {result.get('correct_count', '?')}/"
+            f"{result.get('total_attempted', '?')}\n"
+            f"Answered: {answered}/{total}\n"
+            f"Timeouts: {timed_out}\n"
+            f"Errors: {errors}\n"
+            f"{result.get('message', '')}"
         )
-        print(final_status)
-        return final_status, pd.DataFrame(results_log)
 
-    except requests.exceptions.HTTPError as e:
-        try:
-            error_detail = e.response.json()
-        except Exception:
-            error_detail = e.response.text
-        msg = f"❌ Submission Failed (HTTP {e.response.status_code}):\n{error_detail}"
-        print(msg)
-        return msg, pd.DataFrame(results_log)
+        return status, df
 
     except Exception as e:
-        msg = f"❌ Submission Failed: {e}"
-        print(msg)
-        return msg, pd.DataFrame(results_log)
+        fail = (
+            f"Submission failed:\n{e}\n\n"
+            f"Answered: {answered}/{total}\n"
+            f"Timeouts: {timed_out}\n"
+            f"Errors: {errors}"
+        )
+
+        return fail, df
 
 
 # ============================================================
-# GRADIO UI
+# UI
 # ============================================================
 
 with gr.Blocks() as demo:
-    gr.Markdown("# 🤖 LangGraph GAIA Benchmark Agent")
-
-    gr.Markdown("""
-    **Strategy:**
-    - ✅ Questions sorted: Simple → Medium → Hard
-    - ⏰ Timeouts: Simple=60s | Medium=90s | Hard=120s
-    - 🎯 Target: 30%+ success rate on simple questions first
-    - 🔄 Model stack: Qwen3 32B → Llama 3.3 70B → Llama 3.1 8B
-
-    **Instructions:**
-    1. Login with Hugging Face
-    2. Click **Run Evaluation**
-    3. Agent answers all questions and submits automatically
-    """)
+    gr.Markdown("# Benchmark Agent Submission Tool")
+
+    gr.Markdown(
+        """
+### Current Strategy
+
+- Simple -> Medium -> Hard
+- Fast model routing in backend
+- Auto timeouts
+- Auto submit
+- CSV logs saved
+        """
+    )
 
     gr.LoginButton()
 
-    run_button = gr.Button("Run Evaluation and Submit All Answers", variant="primary", size="lg")
+    run_btn = gr.Button(
+        "Run Evaluation and Submit",
+        variant="primary",
+        size="lg"
+    )
 
-    status_output = gr.Textbox(
-        label="Submission Status",
-        lines=8,
+    status_box = gr.Textbox(
+        label="Status",
+        lines=12,
         interactive=False
     )
 
     results_table = gr.DataFrame(
-        label="Agent Performance per Question",
+        label="Per Question Results",
         wrap=True
     )
 
-    run_button.click(
+    run_btn.click(
         fn=run_and_submit_all,
-        outputs=[status_output, results_table]
+        outputs=[status_box, results_table]
     )
 
+# ============================================================
+# MAIN
+# ============================================================
+
 if __name__ == "__main__":
     demo.launch(debug=True)