Evaluation 5

app.py

@@ -1,143 +1,155 @@
-import os
-import json
-import random
-import requests
-import uvicorn
-
-from fastapi import FastAPI, HTTPException
-from fastapi.responses import FileResponse
+import os, threading
 import gradio as gr
-import pandas as pd
-
+from crew import run_parallel_crew
 from crew import run_crew
-
-# ─── Configuration ─────────────────────────────────────────────────────────
-PORT       = int(os.getenv("PORT", 7860))
-LOCAL_API  = f"http://127.0.0.1:{PORT}"
-SPACE_ID   = os.getenv("SPACE_ID", "Psiska/General_AI_Assistant")
-QUESTIONS_PATH = os.getenv("QUESTIONS_PATH", "data/gaia_validation.jsonl")
-
-# ─── FastAPI setup ──────────────────────────────────────────────────────────
-api = FastAPI(title="GAIA Evaluation API")
-
-# Load questions from JSONL
-questions = []
-with open(QUESTIONS_PATH, 'r') as f:
-    for line in f:
-        questions.append(json.loads(line))
-
-@api.get("/questions")
-def get_questions():
-    return questions
-
-@api.get("/random-question")
-def get_random_question():
-    return random.choice(questions)
-
-@api.get("/files/{task_id}")
-def get_file(task_id: str):
-    entry = next(
-        (q for q in questions if str(q.get("task_id") or q.get("id")) == task_id),
-        None
-    )
-    if not entry or not entry.get("file_name"):
-        raise HTTPException(status_code=404, detail="File not found for this task")
-    file_path = os.path.join("data", entry["file_name"])
-    if not os.path.exists(file_path):
-        raise HTTPException(status_code=404, detail="File missing on disk")
-    return FileResponse(file_path)
-
-@api.post("/submit")
-def submit(batch: dict):
-    username   = batch.get("username", "")
-    agent_code = batch.get("agent_code", "")
-    answers    = batch.get("answers", [])
-
-    total    = len(answers)
-    correct  = 0
-
-    # Map task_id -> ground-truth
-    truth_map = {
-        str(q.get("task_id") or q.get("id")): str(q.get("Final answer") or q.get("final_answer") or "")
-        for q in questions
-    }
-    for ans in answers:
-        tid = str(ans.get("task_id"))
-        if str(ans.get("submitted_answer", "")) == truth_map.get(tid, ""):
-            correct += 1
-
-    score = round(100 * correct / total) if total else 0
-    return {
-        "username":       username,
-        "agent_code":     agent_code,
-        "score":          score,
-        "correct_count":  correct,
-        "total_attempted": total
-    }
-
-# ─── Gradio UI setup ─────────────────────────────────────────────────────────
-def run_and_submit_all(username: str):
-    if not username:
-        return "🔒 Please enter your Hugging Face username.", None
-    try:
-        # Fetch questions from local API
-        resp = requests.get(f"{LOCAL_API}/questions", timeout=15)
-        resp.raise_for_status()
-        qs = resp.json()
-
-        logs, payload = [], []
-        for q in qs:
-            task_id  = str(q.get("task_id") or q.get("id"))
-            question = q.get("question", "")
-            file_name= q.get("file_name", "")
-
-            # Download file if exists
-            if file_name:
-                file_resp = requests.get(f"{LOCAL_API}/files/{task_id}", timeout=15)
-                file_resp.raise_for_status()
-                os.makedirs("data", exist_ok=True)
-                path = os.path.join("data", file_name)
-                with open(path, "wb") as fd:
-                    fd.write(file_resp.content)
-
+from utils import get_questions
+
+
+def ask(question, openai_api_key, gemini_api_key, anthropic_api_key, file_name = ""):
+    """
+    Ask General AI Assistant a question to answer.
+
+    Args:
+        question (str): The question to answer
+        openai_api_key (str): OpenAI API key
+        gemini_api_key (str): Gemini API key
+        anthropic_api_key (str): Anthropic API key
+        file_name (str): Optional file name
+
+    Returns:
+        str: The answer to the question
+    """
+    if not question:
+        raise gr.Error("Question is required.")
+
+    if not openai_api_key:
+        raise gr.Error("OpenAI API Key is required.")
+    
+    if not gemini_api_key:
+        raise gr.Error("Gemini API Key is required.")
+
+    if not anthropic_api_key:
+        raise gr.Error("Anthropic API Key is required.")
+        
+    if file_name:
+        file_name = f"data/{file_name}"
+    
+    lock = threading.Lock()
+
+    with lock:
+        answer = ""
+
+        try:
+            os.environ["OPENAI_API_KEY"] = openai_api_key
+            os.environ["GEMINI_API_KEY"] = gemini_api_key
+            os.environ["MODEL_API_KEY"] = anthropic_api_key
+            
+            #answer = run_parallel_crew(question, file_name)
             answer = run_crew(question, file_name)
-            payload.append({"task_id": task_id, "submitted_answer": answer})
-            logs.append({"Task ID": task_id, "Question": question, "Answer": answer})
-
-        submission = {
-            "username":   username,
-            "agent_code": f"https://huggingface.co/spaces/{SPACE_ID}/tree/main",
-            "answers":    payload
-        }
-        sub_resp = requests.post(f"{LOCAL_API}/submit", json=submission, timeout=60)
-        sub_resp.raise_for_status()
-        result = sub_resp.json()
-
-        status = (
-            f"✅ {result['username']} scored {result['score']}% "
-            f"({result['correct_count']}/{result['total_attempted']} correct)"
+        except Exception as e:
+            raise gr.Error(e)
+        finally:
+            del os.environ["OPENAI_API_KEY"]
+            del os.environ["GEMINI_API_KEY"]
+            del os.environ["MODEL_API_KEY"]
+        
+        return answer
+
+gr.close_all()
+
+with gr.Blocks() as grady:
+    gr.Markdown("## Grady - General AI Assistant")
+
+    with gr.Tab("Solution"):
+        gr.Markdown(os.environ.get("DESCRIPTION"))
+
+        with gr.Row():
+            with gr.Column(scale=3):
+                with gr.Row():
+                    question = gr.Textbox(
+                        label="Question *",
+                        placeholder="In the 2025 Gradio Agents & MCP Hackathon, what percentage of participants submitted a solution during the last 24 hours?",
+                        interactive=True
+                    )
+                with gr.Row():
+                    level = gr.Radio(
+                        choices=[1, 2, 3],
+                        label="GAIA Benchmark Level",
+                        interactive=True,
+                        scale=1
+                    )
+                    ground_truth = gr.Textbox(
+                        label="Ground Truth",
+                        interactive=True,
+                        scale=1
+                    )
+                    file_name = gr.Textbox(
+                        label="File Name",
+                        interactive=True,
+                        scale=2
+                    )
+                with gr.Row():
+                    openai_api_key = gr.Textbox(
+                        label="OpenAI API Key *",
+                        type="password",
+                        placeholder="sk‑...",
+                        interactive=True
+                    )
+                    gemini_api_key = gr.Textbox(
+                        label="Gemini API Key *",
+                        type="password",
+                        interactive=True
+                    )
+                    anthropic_api_key = gr.Textbox(
+                        label="Anthropic API Key *",
+                        type="password",
+                        placeholder="sk-ant-...",
+                        interactive=True
+                    )
+                with gr.Row():
+                    clear_btn = gr.ClearButton(
+                        components=[question, level, ground_truth, file_name]
+                    )
+                    submit_btn = gr.Button("Submit", variant="primary")
+            with gr.Column(scale=1):
+                answer = gr.Textbox(
+                    label="Answer",
+                    lines=1,
+                    interactive=False
+                )
+    
+        submit_btn.click(
+            fn=ask,
+            inputs=[question, openai_api_key, gemini_api_key, anthropic_api_key, file_name],
+            outputs=answer
         )
-        return status, pd.DataFrame(logs)
-    except Exception as e:
-        return f"❌ Error: {e}", None
-
-# Build Gradio interface
-with gr.Blocks(title="GAIA Evaluation Runner") as demo:
-    gr.Markdown("# GAIA Evaluation Runner")
-    user_input = gr.Textbox(label="Hugging Face Username")
-    run_btn    = gr.Button("Run & Submit All Answers")
-    status     = gr.Textbox(label="Status", interactive=False)
-    table      = gr.DataFrame(headers=["Task ID", "Question", "Answer"], label="Log of Q&A")
-
-    run_btn.click(
-        fn=run_and_submit_all,
-        inputs=[user_input],
-        outputs=[status, table]
-    )
-
-# Mount Gradio app on FastAPI
-api.mount("/", demo)
-
-if __name__ == "__main__":
-    uvicorn.run(api, host="0.0.0.0", port=PORT)
+    
+        QUESTION_FILE_PATH = "data/gaia_validation.jsonl"
+        
+        gr.Examples(
+            label="GAIA Benchmark Level 1 Problems",
+            examples=get_questions(QUESTION_FILE_PATH, 1),
+            inputs=[question, level, ground_truth, file_name, openai_api_key, gemini_api_key, anthropic_api_key],
+            outputs=answer,
+            cache_examples=False
+        )
+    
+        gr.Examples(
+            label="GAIA Benchmark Level 2 Problems",
+            examples=get_questions(QUESTION_FILE_PATH, 2),
+            inputs=[question, level, ground_truth, file_name, openai_api_key, gemini_api_key, anthropic_api_key],
+            outputs=answer,
+            cache_examples=False
+        )
+    
+        gr.Examples(
+            label="GAIA Benchmark Level 3 Problems",
+            examples=get_questions(QUESTION_FILE_PATH, 3),
+            inputs=[question, level, ground_truth, file_name, openai_api_key, gemini_api_key, anthropic_api_key],
+            outputs=answer,
+            cache_examples=False
+        )
+    with gr.Tab("Documentation"):
+        gr.Markdown(os.environ.get("DOCUMENTATION"))
 
+grady.launch(mcp_server=True)