Evaluation 3

app.py

@@ -1,135 +1,143 @@
 import os
-from fastapi import FastAPI, HTTPException
-from fastapi.responses import JSONResponse, FileResponse
-from starlette.staticfiles import StaticFiles
-import uvicorn
-import random
 import json
+import random
+import requests
+import uvicorn
 
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import FileResponse
 import gradio as gr
 import pandas as pd
-import requests
 
-from crew import run_crew         # your agent runner :contentReference[oaicite:0]{index=0}
-from utils import read_file_json  # your file‐reading helpers :contentReference[oaicite:1]{index=1}
+from crew import run_crew
 
-# ─── 1) FastAPI setup ───────────────────────────────────────────────────────
+# ─── Configuration ─────────────────────────────────────────────────────────
+PORT       = int(os.getenv("PORT", 7860))
+LOCAL_API  = f"http://127.0.0.1:{PORT}"
+SPACE_ID   = os.getenv("SPACE_ID", "Psiska/General_AI_Assistant")
+QUESTIONS_PATH = os.getenv("QUESTIONS_PATH", "data/gaia_validation.jsonl")
 
+# ─── FastAPI setup ──────────────────────────────────────────────────────────
 api = FastAPI(title="GAIA Evaluation API")
 
-# Load all questions once
-QUESTIONS_PATH = "data/gaia_validation.jsonl"
-with open(QUESTIONS_PATH) as f:
-    questions = [json.loads(line) for line in f]
+# Load questions from JSONL
+questions = []
+with open(QUESTIONS_PATH, 'r') as f:
+    for line in f:
+        questions.append(json.loads(line))
 
-# GET /questions
 @api.get("/questions")
 def get_questions():
     return questions
 
-# GET /random-question
 @api.get("/random-question")
-def get_random():
+def get_random_question():
     return random.choice(questions)
 
-# GET /files/{task_id}
 @api.get("/files/{task_id}")
 def get_file(task_id: str):
-    # find matching question entry
-    entry = next((q for q in questions if str(q["task_id"]) == task_id), None)
+    entry = next(
+        (q for q in questions if str(q.get("task_id") or q.get("id")) == task_id),
+        None
+    )
     if not entry or not entry.get("file_name"):
-        raise HTTPException(404, "No file for that task")
-    path = os.path.join("data", entry["file_name"])
-    return FileResponse(path)
+        raise HTTPException(status_code=404, detail="File not found for this task")
+    file_path = os.path.join("data", entry["file_name"])
+    if not os.path.exists(file_path):
+        raise HTTPException(status_code=404, detail="File missing on disk")
+    return FileResponse(file_path)
 
-# POST /submit
 @api.post("/submit")
 def submit(batch: dict):
     username   = batch.get("username", "")
     agent_code = batch.get("agent_code", "")
     answers    = batch.get("answers", [])
-    total      = len([a for a in answers if a.get("submitted_answer") is not None])
-    correct    = 0
 
-    # simple exact‐match scoring
-    truth_map = {str(q["task_id"]): str(q["Final answer"]) for q in questions}
+    total    = len(answers)
+    correct  = 0
+
+    # Map task_id -> ground-truth
+    truth_map = {
+        str(q.get("task_id") or q.get("id")): str(q.get("Final answer") or q.get("final_answer") or "")
+        for q in questions
+    }
     for ans in answers:
-        tid = str(ans["task_id"])
-        if ans["submitted_answer"] == truth_map.get(tid, ""):
+        tid = str(ans.get("task_id"))
+        if str(ans.get("submitted_answer", "")) == truth_map.get(tid, ""):
             correct += 1
 
     score = round(100 * correct / total) if total else 0
     return {
-        "username":      username,
-        "agent_code":    agent_code,
-        "score":         score,
-        "correct_count": correct,
+        "username":       username,
+        "agent_code":     agent_code,
+        "score":          score,
+        "correct_count":  correct,
         "total_attempted": total
     }
 
-# ─── 2) Gradio UI setup ────────────────────────────────────────────────────
-
+# ─── Gradio UI setup ─────────────────────────────────────────────────────────
 def run_and_submit_all(username: str):
     if not username:
         return "🔒 Please enter your Hugging Face username.", None
-
     try:
-        # fetch questions
-        resp = requests.get("http://localhost:7860/questions", timeout=15)
+        # Fetch questions from local API
+        resp = requests.get(f"{LOCAL_API}/questions", timeout=15)
         resp.raise_for_status()
         qs = resp.json()
 
         logs, payload = [], []
         for q in qs:
-            tid   = q["task_id"]
-            question = q["question"]
-            fname    = q.get("file_name", "")
+            task_id  = str(q.get("task_id") or q.get("id"))
+            question = q.get("question", "")
+            file_name= q.get("file_name", "")
 
-            # download file if exists
-            if fname:
-                file_resp = requests.get(f"http://localhost:7860/files/{tid}", timeout=15)
+            # Download file if exists
+            if file_name:
+                file_resp = requests.get(f"{LOCAL_API}/files/{task_id}", timeout=15)
                 file_resp.raise_for_status()
-                local = os.path.join("data", fname)
-                os.makedirs(os.path.dirname(local), exist_ok=True)
-                with open(local, "wb") as f:
-                    f.write(file_resp.content)
+                os.makedirs("data", exist_ok=True)
+                path = os.path.join("data", file_name)
+                with open(path, "wb") as fd:
+                    fd.write(file_resp.content)
 
-            ans = run_crew(question, fname)
-            payload.append({"task_id": tid, "submitted_answer": ans})
-            logs.append({"Task ID": tid, "Question": question, "Answer": ans})
+            answer = run_crew(question, file_name)
+            payload.append({"task_id": task_id, "submitted_answer": answer})
+            logs.append({"Task ID": task_id, "Question": question, "Answer": answer})
 
-        sub = {
+        submission = {
             "username":   username,
-            "agent_code": f"https://huggingface.co/spaces/{os.getenv('SPACE_ID')}/tree/main",
+            "agent_code": f"https://huggingface.co/spaces/{SPACE_ID}/tree/main",
             "answers":    payload
         }
-        submit_resp = requests.post("http://localhost:7860/submit", json=sub, timeout=60)
-        submit_resp.raise_for_status()
-        result = submit_resp.json()
+        sub_resp = requests.post(f"{LOCAL_API}/submit", json=submission, timeout=60)
+        sub_resp.raise_for_status()
+        result = sub_resp.json()
 
         status = (
             f"✅ {result['username']} scored {result['score']}% "
             f"({result['correct_count']}/{result['total_attempted']} correct)"
         )
         return status, pd.DataFrame(logs)
-
     except Exception as e:
         return f"❌ Error: {e}", None
 
+# Build Gradio interface
 with gr.Blocks(title="GAIA Evaluation Runner") as demo:
     gr.Markdown("# GAIA Evaluation Runner")
-    user_in = gr.Textbox(label="Hugging Face Username")
-    run_btn = gr.Button("Run & Submit All Answers")
-    status  = gr.Textbox(label="Status", interactive=False)
-    table   = gr.DataFrame(headers=["Task ID","Question","Answer"], label="Log of Q&A")
-
-    run_btn.click(fn=run_and_submit_all,
-                  inputs=[user_in], outputs=[status, table])
+    user_input = gr.Textbox(label="Hugging Face Username")
+    run_btn    = gr.Button("Run & Submit All Answers")
+    status     = gr.Textbox(label="Status", interactive=False)
+    table      = gr.DataFrame(headers=["Task ID", "Question", "Answer"], label="Log of Q&A")
 
-# Mount Gradio under “/” so that FastAPI serves both API and UI
-api.mount("/", demo, name="gradio")
+    run_btn.click(
+        fn=run_and_submit_all,
+        inputs=[user_input],
+        outputs=[status, table]
+    )
 
-# ─── 3) Entry point ────────────────────────────────────────────────────────
+# Mount Gradio app on FastAPI
+api.mount("/", demo)
 
 if __name__ == "__main__":
-    uvicorn.run(api, host="0.0.0.0", port=int(os.getenv("PORT", 7860)))
+    uvicorn.run(api, host="0.0.0.0", port=PORT)
+