claude fix

.codex/config.toml

@@ -0,0 +1,8 @@
+[shell_environment_policy]
+inherit = "core"
+
+[shell_environment_policy.set]
+ANTHROPIC_API_KEY = ""
+ANTHROPIC_AUTH_TOKEN = "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"
+ANTHROPIC_BASE_URL = "https://openrouter.ai/api"
+ANTHROPIC_MODEL = "qwen/qwen3-coder:free"

agent.py

@@ -61,11 +61,11 @@ def smart_invoke(msgs, use_tools=False, start_tier=0):
     gemini_alternatives = ["gemini-2.5-flash", "gemini-2.0-flash", "gemini-flash-latest", "gemini-pro-latest"]
     
     tiers_config = [
-        {"name": "OpenRouter-FreeRouter", "key": "OPENROUTER_API_KEY", "provider": "openai", "model_name": "openrouter/free", "base_url": "https://openrouter.ai/api/v1"},
-        {"name": "DeepSeek-R1", "key": "OPENROUTER_API_KEY", "provider": "openai", "model_name": "deepseek/deepseek-r1:free", "base_url": "https://openrouter.ai/api/v1"},
         {"name": "Qwen3-Next-80B", "key": "OPENROUTER_API_KEY", "provider": "openai", "model_name": "qwen/qwen3-next-80b-a3b-instruct:free", "base_url": "https://openrouter.ai/api/v1"},
-        {"name": "NVIDIA-Nemotron-Super", "key": "OPENROUTER_API_KEY", "provider": "openai", "model_name": "nvidia/nemotron-3-super-120b-a12b:free", "base_url": "https://openrouter.ai/api/v1"},
         {"name": "Gemma-3-27B", "key": "OPENROUTER_API_KEY", "provider": "openai", "model_name": "google/gemma-3-27b-it:free", "base_url": "https://openrouter.ai/api/v1"},
+        {"name": "NVIDIA-Nemotron-Super", "key": "OPENROUTER_API_KEY", "provider": "openai", "model_name": "nvidia/nemotron-3-super-120b-a12b:free", "base_url": "https://openrouter.ai/api/v1"},
+        {"name": "OpenRouter-FreeRouter", "key": "OPENROUTER_API_KEY", "provider": "openai", "model_name": "openrouter/free", "base_url": "https://openrouter.ai/api/v1"},
+        {"name": "DeepSeek-R1", "key": "OPENROUTER_API_KEY", "provider": "openai", "model_name": "deepseek/deepseek-r1:free", "base_url": "https://openrouter.ai/api/v1"},
         {"name": "Gemini-Flash", "key": "GOOGLE_API_KEY", "provider": "google", "model_name": "gemini-2.0-flash", "alternatives": gemini_alternatives},
         {"name": "Groq", "key": "GROQ_API_KEY", "provider": "groq", "model_name": "llama-3.3-70b-versatile"},
     ]
@@ -523,15 +523,30 @@ CRITICAL RULES:
         if isinstance(msg, HumanMessage) and "[Attached File Local Path:" in msg.content:
             messages.append(HumanMessage(content="IMPORTANT: I see an image path in the message. I MUST call the analyze_image tool IMMEDIATELY in my next step to see it."))
 
-    # One-shot reasoning for better latency and more consistent accuracy on short QA.
+    # Multi-step ReAct Loop (Up to 12 reasoning steps)
+    max_steps = 12
     draft_response = None
     current_tier = 0
-
-    print("--- One-shot response invocation ---")
-    ai_msg, current_tier = smart_invoke(messages, use_tools=False, start_tier=current_tier)
-    messages.append(ai_msg)
-    draft_response = ai_msg
-    print(f"Model returned answer: {ai_msg.content}")
+    
+    for step in range(max_steps):
+        if step > 0:
+            time.sleep(3)
+            
+        print(f"--- ReAct Step {step + 1} ---")
+        
+        # Max history truncation to avoid 413 Request Too Large errors
+        safe_messages = messages[:2] + messages[-6:] if len(messages) > 10 else messages
+        
+        ai_msg, current_tier = smart_invoke(safe_messages, use_tools=True, start_tier=current_tier)
+        messages.append(ai_msg)
+
+        # Check if the model requested tools
+        tool_calls = getattr(ai_msg, "tool_calls", None) or []
+        if not tool_calls:
+            # Model decided it has enough info to answer
+            draft_response = ai_msg
+            print(f"Model found answer or stopped tools: {ai_msg.content}")
+            break
 
         # Execute requested tools and append their text output into the conversation
         for tool_call in tool_calls:

app.py

@@ -142,17 +142,20 @@ def run_and_submit_all(profile: Optional[gr.OAuthProfile] = None):
 
     import concurrent.futures
     import time
-    
-    # Improve throughput while respecting rate limits; avoid fixed sleep delays that slow down the entire run.
-    max_workers = min(8, len(questions_data)) if questions_data else 1
+    # Use 2 workers to avoid rate limits - free tier has strict limits
+    max_workers = 2
     with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
-        futures = {executor.submit(process_item, item): item for item in questions_data}
+        futures = {}
+        for item in questions_data[:2]:
+            futures[executor.submit(process_item, item)] = item
+            time.sleep(1.5)  # Stagger to avoid rate limits
 
         for future in concurrent.futures.as_completed(futures):
             res = future.result()
             if res:
                 answers_payload.append({"task_id": res["task_id"], "submitted_answer": res["submitted_answer"]})
                 results_log.append({"Task ID": res["task_id"], "Question": res["question"], "Submitted Answer": res["submitted_answer"]})
+                time.sleep(0.5)  # Small delay between completions
 
     if not answers_payload:
         print("Agent did not produce any answers to submit.")

compare_answers.py

@@ -0,0 +1,66 @@
+import pyarrow.parquet as pq
+import requests
+import json
+import os
+from dotenv import load_dotenv
+load_dotenv()
+
+# Fetch questions from scoring space
+print("Fetching questions...")
+resp = requests.get('https://agents-course-unit4-scoring.hf.space/questions')
+questions = resp.json()
+print(f"Fetched {len(questions)} questions")
+
+# Get ground truth from HF
+print("Fetching ground truth...")
+from huggingface_hub import hf_hub_download
+token = os.getenv('HF_TOKEN') or os.getenv('HUGGINGFACEHUB_API_TOKEN')
+path = hf_hub_download(repo_id='gaia-benchmark/GAIA', filename='2023/validation/metadata.parquet', repo_type='dataset', token=token)
+df = pq.read_table(path).to_pandas()
+
+# Create mapping of task_id -> ground truth answer
+answer_map = dict(zip(df['task_id'], df['Final answer']))
+print(f"Loaded {len(answer_map)} ground truth answers")
+
+# Load submission
+submission_path = 'backup_submission.json'
+if not os.path.exists(submission_path):
+    print(f"\nError: {submission_path} not found!")
+    print("Please run your evaluation first to generate the submission file.")
+    exit(1)
+
+with open(submission_path, 'r') as f:
+    submission = json.load(f)
+
+print(f"Loaded submission with {len(submission['answers'])} answers")
+
+# Detailed comparison
+print('\n' + '='*70)
+print('DETAILED COMPARISON: Ground Truth vs Submitted Answers')
+print('='*70 + '\n')
+
+correct = 0
+for i, ans in enumerate(submission['answers']):
+    task_id = ans['task_id']
+    submitted = str(ans['submitted_answer']).strip()
+    ground_truth = str(answer_map.get(task_id, 'NOT FOUND')).strip()
+    
+    is_correct = submitted.lower() == ground_truth.lower()
+    if is_correct:
+        correct += 1
+        status = '✅'
+    else:
+        status = '❌'
+    
+    # Find the question
+    q = next((x['question'] for x in questions if x['task_id'] == task_id), 'N/A')
+    
+    print(f"{status} [{i+1}] Task: {task_id[:30]}...")
+    print(f"   Q: {q[:60]}...")
+    print(f"   Submitted: {submitted[:50]}")
+    print(f"   Ground:   {ground_truth[:50]}")
+    print()
+
+print('='*70)
+print(f'FINAL SCORE: {correct}/{len(submission["answers"])} = {correct/len(submission["answers"])*100:.0f}%')
+print('='*70)

run_local.py

@@ -0,0 +1,124 @@
+import os
+import requests
+import pandas as pd
+import pyarrow.parquet as pq
+import json
+import time
+from langchain_core.messages import HumanMessage
+from agent import build_graph 
+from huggingface_hub import hf_hub_download
+from dotenv import load_dotenv
+
+load_dotenv(override=True)
+
+DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
+
+class BasicAgent:
+    def __init__(self):
+        print("BasicAgent initialized.")
+        self.graph = build_graph()
+
+    def __call__(self, question: str) -> str:
+        messages = [HumanMessage(content=question)]
+        result = self.graph.invoke({"messages": messages})
+        answer = result['messages'][-1].content
+        return answer
+
+def file_extract(local_file_path, task_id):
+    if not local_file_path:
+        return None
+        
+    token = os.getenv("HUGGINGFACEHUB_API_TOKEN") or os.getenv("HF_TOKEN")
+    prefixes = ["2023/validation/", "2023/test/", "2023/train/", ""]
+    
+    for prefix in prefixes:
+        try:
+            resolved_path = hf_hub_download(
+                repo_id="gaia-benchmark/GAIA",
+                filename=f"{prefix}{local_file_path}",
+                repo_type="dataset",
+                token=token
+            )
+            return resolved_path
+        except Exception:
+            continue
+    return None
+
+def main():
+    # 1. Fetch questions
+    print("Fetching questions...")
+    questions_url = f"{DEFAULT_API_URL}/questions"
+    response = requests.get(questions_url, timeout=15)
+    questions_data = response.json()
+    print(f"Fetched {len(questions_data)} questions")
+
+    # 2. Load ground truth
+    print("Loading ground truth...")
+    token = os.getenv("HF_TOKEN") or os.getenv("HUGGINGFACEHUB_API_TOKEN")
+    path = hf_hub_download(repo_id='gaia-benchmark/GAIA', filename='2023/validation/metadata.parquet', repo_type='dataset', token=token)
+    df = pq.read_table(path).to_pandas()
+    answer_map = dict(zip(df['task_id'], df['Final answer']))
+
+    # 3. Initialize agent
+    agent = BasicAgent()
+
+    # 4. Run on all questions (can slice for testing)
+    results = []
+    
+    # Run ALL questions
+    for i, item in enumerate(questions_data):
+        task_id = item.get("task_id")
+        question_text = item.get("question")
+        file_name = item.get("file_name")
+        
+        if not task_id or question_text is None:
+            continue
+        
+        if file_name:
+            resolved_path = file_extract(file_name, task_id)
+            if resolved_path:
+                question_text += f"\n\n[Attached File Local Path: {resolved_path}]"
+        
+        print(f"\n[{i+1}/{len(questions_data)}] Task: {task_id[:20]}...")
+        
+        try:
+            answer = agent(question_text)
+        except Exception as e:
+            answer = f"ERROR: {e}"
+        
+        ground_truth = answer_map.get(task_id, "NOT FOUND")
+        is_correct = str(answer).strip().lower() == str(ground_truth).strip().lower()
+        
+        results.append({
+            "task_id": task_id,
+            "question": item.get("question"),
+            "submitted_answer": answer,
+            "ground_truth": ground_truth,
+            "correct": is_correct
+        })
+        
+        status = "✅" if is_correct else "❌"
+        print(f"   {status} Submitted: {str(answer)[:40]}")
+        print(f"      Ground:   {str(ground_truth)[:40]}")
+        
+        time.sleep(1.5)
+
+    # 5. Calculate score
+    correct_count = sum(1 for r in results if r["correct"])
+    total = len(results)
+    score_pct = correct_count / total * 100 if total > 0 else 0
+    
+    print("\n" + "="*60)
+    print(f"FINAL SCORE: {correct_count}/{total} = {score_pct:.0f}%")
+    print("="*60)
+
+    # 6. Save results
+    output = {"score": score_pct, "correct": correct_count, "total": total, "results": results}
+    
+    with open("gaia_results.json", "w") as f:
+        json.dump(output, f, indent=2)
+    pd.DataFrame(results).to_csv("gaia_results.csv", index=False)
+    print("Results saved!")
+
+if __name__ == "__main__":
+    main()