Update app.py

app.py

@@ -1,6 +1,5 @@
 # ==========================================================
-# UPDATED app.py
-# Observability + Model Logs + Adaptive Recursion
+# app.py — Flat routing, uniform timeout + recursion for all questions
 # ==========================================================
 
 import os
@@ -17,57 +16,39 @@ from agent import (
     build_graph,
     extract_final_answer,
     extract_tools_used,
-    classify_question,
     get_last_trace,
 )
 
 # ==========================================================
-# LIVE LOGGING
+# CONFIG
 # ==========================================================
 
 sys.stdout.reconfigure(line_buffering=True)
 
-# ==========================================================
-# CONFIG
-# ==========================================================
-
 DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 
-TIMEOUTS = {
-    "simple": 20,
-    "medium": 35,
-    "hard": 55,
-}
-
-PAUSE = {
-    "simple": 0.5,
-    "medium": 1.0,
-    "hard": 1.5,
-}
-
-RECURSION_LIMITS = {
-    "simple": 6,
-    "medium": 10,
-    "hard": 14,
-}
+# Single values — no per-difficulty branching
+TIMEOUT_SECONDS  = 60
+RECURSION_LIMIT  = 12
+PAUSE_SECONDS    = 1.0
 
 # ==========================================================
-# LOGGER
+# LOGGING
 # ==========================================================
 
-def log(msg):
+def log(msg: str) -> None:
     print(msg, flush=True)
 
 # ==========================================================
-# THREAD TIMEOUT
+# TIMEOUT WRAPPER
 # ==========================================================
 
-def run_with_timeout(fn, timeout_seconds):
-    state = {
-        "done": False,
-        "result": None,
-        "error": None
-    }
+def run_with_timeout(fn, timeout_seconds: int):
+    """
+    Run fn() in a daemon thread.
+    Returns (result, timed_out). Raises if fn raised.
+    """
+    state = {"done": False, "result": None, "error": None}
 
     def target():
         try:
@@ -82,11 +63,9 @@ def run_with_timeout(fn, timeout_seconds):
     thread.join(timeout_seconds)
 
     if not state["done"]:
-        return None, True
-
+        return None, True          # timed out
     if state["error"]:
         raise state["error"]
-
     return state["result"], False
 
 # ==========================================================
@@ -95,331 +74,209 @@ def run_with_timeout(fn, timeout_seconds):
 
 class BenchmarkAgent:
     def __init__(self):
-        log("Initializing agent...")
+        log("Initializing agent graph...")
         self.graph = build_graph()
         log("Agent ready.")
 
-    def __call__(self, question: str):
-        difficulty = classify_question(question)
-        recursion_limit = RECURSION_LIMITS[difficulty]
-
+    def __call__(self, question: str) -> tuple[str, list, dict]:
         result = self.graph.invoke(
             {"messages": [HumanMessage(content=question)]},
-            {"recursion_limit": recursion_limit}
+            {"recursion_limit": RECURSION_LIMIT},
         )
-
-        answer = extract_final_answer(result)
-        tools = extract_tools_used(result)
-        trace = get_last_trace()
-
-        if not answer:
-            answer = "N/A"
-
+        answer = extract_final_answer(result) or "N/A"
+        tools  = extract_tools_used(result)
+        trace  = get_last_trace()
         return answer, tools, trace
 
 # ==========================================================
-# HELPERS
+# API HELPERS
 # ==========================================================
 
-def fetch_questions(url):
+def fetch_questions(url: str) -> list[dict]:
     r = requests.get(url, timeout=20)
     r.raise_for_status()
     return r.json()
 
 
-def submit_answers(url, payload):
+def submit_answers(url: str, payload: dict) -> dict:
     r = requests.post(url, json=payload, timeout=60)
     r.raise_for_status()
     return r.json()
 
-
-def sort_questions(items):
-    order = {
-        "simple": 0,
-        "medium": 1,
-        "hard": 2
-    }
-
-    return sorted(
-        items,
-        key=lambda x: order.get(
-            classify_question(
-                x.get("question", "")
-            ),
-            9
-        )
-    )
-
 # ==========================================================
 # MAIN RUNNER
 # ==========================================================
 
 def run_and_submit_all(profile: gr.OAuthProfile | None):
     if not profile:
-        return "Please login first.", None
+        return "Please log in first.", None
 
     space_id = os.getenv("SPACE_ID")
-
     if not space_id:
-        return "SPACE_ID missing.", None
-
-    username = profile.username.strip()
+        return "SPACE_ID environment variable is missing.", None
 
+    username   = profile.username.strip()
     questions_url = f"{DEFAULT_API_URL}/questions"
-    submit_url = f"{DEFAULT_API_URL}/submit"
-
-    agent_code = (
-        f"https://huggingface.co/spaces/"
-        f"{space_id}/tree/main"
-    )
+    submit_url    = f"{DEFAULT_API_URL}/submit"
+    agent_code    = f"https://huggingface.co/spaces/{space_id}/tree/main"
 
     log("=" * 80)
-    log(f"User: {username}")
-    log("Starting benchmark run...")
+    log(f"User     : {username}")
+    log(f"Timeout  : {TIMEOUT_SECONDS}s per question")
+    log(f"Recursion: {RECURSION_LIMIT}")
     log("=" * 80)
 
-    # ------------------------------------------------------
-    # INIT
-    # ------------------------------------------------------
-
+    # --- Init agent ---
     try:
         agent = BenchmarkAgent()
     except Exception as e:
-        return f"Agent init failed: {e}", None
-
-    # ------------------------------------------------------
-    # FETCH QUESTIONS
-    # ------------------------------------------------------
+        return f"Agent initialisation failed: {e}", None
 
+    # --- Fetch questions ---
     try:
         questions = fetch_questions(questions_url)
     except Exception as e:
-        return f"Failed to fetch questions: {e}", None
-
-    questions = sort_questions(questions)
+        return f"Could not fetch questions: {e}", None
 
-    total = len(questions)
-
-    answers = []
-    logs = []
-
-    answered = 0
+    total         = len(questions)
+    answers       = []
+    logs          = []
+    answered      = 0
     timeout_count = 0
-    error_count = 0
-
-    # ------------------------------------------------------
-    # LOOP
-    # ------------------------------------------------------
+    error_count   = 0
 
+    # --- Question loop ---
     for idx, item in enumerate(questions, start=1):
-        task_id = item.get("task_id")
+        task_id  = item.get("task_id", "")
         question = item.get("question", "")
 
-        difficulty = classify_question(question)
-        timeout = TIMEOUTS[difficulty]
-        recursion_limit = RECURSION_LIMITS[difficulty]
-
-        log("\n" + "-" * 80)
-        log(f"Question      : {idx}/{total}")
-        log(f"Task ID       : {task_id}")
-        log(f"Difficulty    : {difficulty.upper()}")
-        log(f"Timeout       : {timeout}s")
-        log(f"Recursion     : {recursion_limit}")
-        log(f"Question Text : {question[:180]}")
+        log("")
+        log("-" * 80)
+        log(f"[{idx}/{total}]  Task: {task_id}")
+        log(f"Question : {question[:200]}")
         log("-" * 80)
 
         start = time.time()
-
         submitted_answer = "N/A"
-        tools_used = []
-        trace = {}
+        tools_used: list[str] = []
+        trace: dict = {}
         status = "UNKNOWN"
 
         try:
             def solve():
                 return agent(question)
 
-            result, did_timeout = run_with_timeout(
-                solve,
-                timeout
-            )
-
-            elapsed = round(
-                time.time() - start,
-                1
-            )
+            result, did_timeout = run_with_timeout(solve, TIMEOUT_SECONDS)
+            elapsed = round(time.time() - start, 1)
 
             if did_timeout:
                 timeout_count += 1
                 status = f"TIMEOUT ({elapsed}s)"
-                submitted_answer = "N/A"
-
             else:
                 submitted_answer, tools_used, trace = result
-
                 if submitted_answer != "N/A":
                     answered += 1
-
                 status = f"OK ({elapsed}s)"
 
         except Exception as e:
-            elapsed = round(
-                time.time() - start,
-                1
-            )
-
+            elapsed = round(time.time() - start, 1)
             error_count += 1
             status = f"ERROR ({elapsed}s)"
-            submitted_answer = "N/A"
-
-            trace = {
-                "model": "N/A",
-                "fallback": "N/A",
-                "model_error": str(e)
-            }
+            trace  = {"model": "N/A", "fallback": "N/A", "model_error": str(e)}
 
-        # --------------------------------------------------
-        # LOG DETAILS
-        # --------------------------------------------------
-
-        log(f"Model Used    : {trace.get('model','N/A')}")
-        log(f"Fallback Used : {trace.get('fallback','N/A')}")
-        log(f"Model Error   : {trace.get('model_error','None')}")
-        log(f"Tools Used    : {', '.join(tools_used) if tools_used else 'None'}")
-        log(f"Answer        : {submitted_answer}")
-        log(f"Status        : {status}")
-
-        # --------------------------------------------------
-        # STORE
-        # --------------------------------------------------
+        log(f"Model    : {trace.get('model', 'N/A')}")
+        log(f"Fallback : {trace.get('fallback', 'N/A')}")
+        log(f"Error    : {trace.get('model_error', 'None')}")
+        log(f"Tools    : {', '.join(tools_used) if tools_used else 'None'}")
+        log(f"Answer   : {submitted_answer}")
+        log(f"Status   : {status}")
 
         answers.append({
             "task_id": task_id,
-            "submitted_answer": str(submitted_answer).strip()
+            "submitted_answer": str(submitted_answer).strip(),
         })
 
         logs.append({
-            "No": idx,
-            "Task ID": task_id,
-            "Difficulty": difficulty.upper(),
-            "Model Used": trace.get("model", "N/A"),
-            "Fallback": trace.get("fallback", "N/A"),
-            "Model Error": trace.get("model_error", "None"),
-            "Tools Used": ", ".join(tools_used),
+            "No":               idx,
+            "Task ID":          task_id,
+            "Model Used":       trace.get("model", "N/A"),
+            "Fallback":         trace.get("fallback", "N/A"),
+            "Model Error":      trace.get("model_error", "None"),
+            "Tools Used":       ", ".join(tools_used),
             "Submitted Answer": submitted_answer,
-            "Status": status,
-            "Question": question[:140]
+            "Status":           status,
+            "Question":         question[:140],
         })
 
-        time.sleep(PAUSE[difficulty])
-
-    # ------------------------------------------------------
-    # SAVE CSV
-    # ------------------------------------------------------
+        time.sleep(PAUSE_SECONDS)
 
+    # --- Save CSV ---
     df = pd.DataFrame(logs)
-
     try:
         df.to_csv("last_run_results.csv", index=False)
-        log("Saved results CSV.")
+        log("Results saved to last_run_results.csv")
     except Exception:
         pass
 
-    # ------------------------------------------------------
-    # SUBMIT
-    # ------------------------------------------------------
-
+    # --- Submit ---
     payload = {
-        "username": username,
+        "username":   username,
         "agent_code": agent_code,
-        "answers": answers
+        "answers":    answers,
     }
 
     try:
-        result = submit_answers(
-            submit_url,
-            payload
-        )
-
+        result = submit_answers(submit_url, payload)
         final_status = (
             f"SUCCESS\n\n"
-            f"User: {result.get('username')}\n"
-            f"Score: {result.get('score','N/A')}%\n"
-            f"Correct: "
-            f"{result.get('correct_count','?')}/"
-            f"{result.get('total_attempted','?')}\n"
+            f"User    : {result.get('username')}\n"
+            f"Score   : {result.get('score', 'N/A')}%\n"
+            f"Correct : {result.get('correct_count', '?')}/{result.get('total_attempted', '?')}\n"
             f"Answered: {answered}/{total}\n"
             f"Timeouts: {timeout_count}\n"
-            f"Errors: {error_count}\n"
-            f"{result.get('message','')}"
+            f"Errors  : {error_count}\n"
+            f"{result.get('message', '')}"
         )
-
         log("=" * 80)
         log(final_status)
         log("=" * 80)
-
         return final_status, df
 
     except Exception as e:
         fail = (
-            f"Submission failed:\n{e}\n\n"
+            f"Submission failed: {e}\n\n"
             f"Answered: {answered}/{total}\n"
             f"Timeouts: {timeout_count}\n"
-            f"Errors: {error_count}"
+            f"Errors  : {error_count}"
         )
-
         return fail, df
 
 # ==========================================================
-# UI
+# GRADIO UI
 # ==========================================================
 
 with gr.Blocks() as demo:
-    gr.Markdown("# Benchmark Agent - Observability Mode")
-
+    gr.Markdown("# Benchmark Agent")
     gr.Markdown(
         """
-### Features
-
-- Adaptive recursion limits
-- Logs selected model per question
-- Logs fallback model usage
-- Logs rate/model errors
-- Logs tools used
-- CSV export
-- Auto submit
+**Model**: `qwen/qwen3-32b` (primary) → `llama-3.3-70b-versatile` → `llama-3.1-8b-instant`  
+**Routing**: All questions use the same path — no classification.  
+**Timeout**: 60 s per question · **Recursion limit**: 12
         """
     )
 
     gr.LoginButton()
 
-    run_btn = gr.Button(
-        "Run Evaluation & Submit",
-        variant="primary",
-        size="lg"
-    )
-
-    status_box = gr.Textbox(
-        label="Status",
-        lines=12
-    )
+    run_btn = gr.Button("Run Evaluation & Submit", variant="primary", size="lg")
 
-    table = gr.DataFrame(
-        label="Per Question Analytics",
-        wrap=True
-    )
+    status_box = gr.Textbox(label="Status", lines=12)
+    table      = gr.DataFrame(label="Per-question results", wrap=True)
 
-    run_btn.click(
-        fn=run_and_submit_all,
-        outputs=[status_box, table]
-    )
+    run_btn.click(fn=run_and_submit_all, outputs=[status_box, table])
 
 # ==========================================================
-# MAIN
+# ENTRY POINT
 # ==========================================================
 
 if __name__ == "__main__":
-    demo.launch(
-        debug=True,
-        ssr_mode=False
-    )
+    demo.launch(debug=True, ssr_mode=False)