""" AgentDebuggerEnv — Interactive Research Showcase & Leaderboard ============================================================= Primary entry point for the Hugging Face Space. Provides a premium, glassmorphic UI to explore model debugging trajectories, benchmark rankings, sandboxed execution, and the technical report. """ import os import sys import json import time import requests import gradio as gr from dotenv import load_dotenv # Load environment variables load_dotenv() # Insert workspace root to path sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # ── Load Evaluation Results or Use Fallback ─────────────────────────────────── EVAL_RESULTS_PATH = "evaluation_results.json" BASE_LEADERBOARD_PATH = "leaderboard/index.html" # Default fallback benchmarks if evaluation_results.json is not present yet DEFAULT_STATS = { "summary": { "overall": { "total": 61, "solved": 41, "solve_rate": 0.672 }, "tiers": { "tier1": { "total": 9, "solved": 9, "solve_rate": 1.0, "mean_reward": 0.92 }, "tier2": { "total": 31, "solved": 24, "solve_rate": 0.774, "mean_reward": 0.68 }, "tier3": { "total": 21, "solved": 8, "solve_rate": 0.381, "mean_reward": 0.34 } } } } def load_evaluation_data(): if os.path.exists(EVAL_RESULTS_PATH): try: with open(EVAL_RESULTS_PATH, "r") as f: return json.load(f) except Exception as e: print(f"Error loading evaluation results: {e}") return {"summary": DEFAULT_STATS["summary"], "results": {}} # Pre-loaded mock trajectories for fallback visualization MOCK_TRAJECTORIES = { "🔢 Off-by-One: binary_search (Tier 1)": { "buggy_code": "def binary_search(arr, target):\n left, right = 0, len(arr)\n while left < right:\n mid = (left + right) // 2\n if arr[mid] == target:\n return mid\n elif arr[mid] < target:\n left = mid + 1\n else:\n right = mid - 1\n return -1", "initial_error": "IndexError: list index out of range when target is the last element", "trajectory": [ { "turn": 1, "observation": "right is initialized to len(arr) on line 2, which is out of bounds for a 0-indexed list of size len(arr). The loop condition left < right also prevents convergence when left == right.", "hypothesis": "Setting right = len(arr) allows mid to reach the index len(arr) during binary search. Python list index bounds require right = len(arr) - 1. Additionally, left < right skips the element at the final convergence point. Both need to be corrected.", "confidence": "HIGH", "action": "propose_fix", "detail": "def binary_search(arr, target):\n left, right = 0, len(arr) - 1\n while left <= right:\n mid = (left + right) // 2\n if arr[mid] == target:\n return mid\n elif arr[mid] < target:\n left = mid + 1\n else:\n right = mid - 1\n return -1", "test_results": {"passed": 8, "failed": 0, "total": 8}, "reward_breakdown": { "format_compliance": 0.10, "hypothesis_quality": 0.20, "localization": 0.15, "fix_quality": 0.35, "semantic_similarity": 0.10, "efficiency_potential": 0.10, "total": 1.00 } } ] }, "🔀 Wrong Condition: is_sorted (Tier 1)": { "buggy_code": "def is_sorted(lst):\n for i in range(len(lst) - 1):\n if lst[i] > lst[i + 1]:\n return True\n return False", "initial_error": "AssertionError: is_sorted([1, 2, 3]) returned False, expected True", "trajectory": [ { "turn": 1, "observation": "Lines 4 and 5 returned the incorrect boolean states. When lst[i] > lst[i+1] is true, the list is unsorted, so it should return False.", "hypothesis": "The return values are inverted. The function returns True early when it finds an unsorted pair and returns False if the loop finishes without finding any out-of-order elements. Swapping True and False fixes this.", "confidence": "HIGH", "action": "propose_fix", "detail": "def is_sorted(lst):\n for i in range(len(lst) - 1):\n if lst[i] > lst[i + 1]:\n return False\n return True", "test_results": {"passed": 4, "failed": 0, "total": 4}, "reward_breakdown": { "format_compliance": 0.10, "hypothesis_quality": 0.20, "localization": 0.15, "fix_quality": 0.35, "semantic_similarity": 0.10, "efficiency_potential": 0.10, "total": 1.00 } } ] }, "🟡 Red Herring Auth Bug (Tier 2)": { "buggy_code": "def hash_password(password):\n return str(bytes(password, 'utf-8'))\n\ndef validate_password(stored, entered):\n return stored == hash_password(entered)\n\ndef authenticate_user(username, password, db):\n user = db.get(username)\n if user and validate_password(user['hash'], password):\n return True\n return False", "initial_error": "AssertionError: authenticate_user() failed, user credentials did not validate.", "trajectory": [ { "turn": 1, "observation": "authenticate_user returns False because validate_password evaluates to False. Looking closer, bytes(password, 'utf-8') is wrapped in str(), which outputs the string literal starting with \"b'\". This makes the hash comparison fail.", "hypothesis": "The bug is in hash_password, not authenticate_user. Calling str(bytes(...)) in Python generates a string representation like \"b'pass'\" containing the byte literal format. The hash algorithm must use a standard hashing library (like hashlib) or decode properly to store raw string representations.", "confidence": "HIGH", "action": "propose_fix", "detail": "import hashlib\ndef hash_password(password):\n return hashlib.sha256(password.encode('utf-8')).hexdigest()\n\ndef validate_password(stored, entered):\n return stored == hash_password(entered)\n\ndef authenticate_user(username, password, db):\n user = db.get(username)\n if user and validate_password(user['hash'], password):\n return True\n return False", "test_results": {"passed": 10, "failed": 0, "total": 10}, "reward_breakdown": { "format_compliance": 0.10, "hypothesis_quality": 0.20, "localization": 0.15, "fix_quality": 0.35, "semantic_similarity": 0.10, "efficiency_potential": 0.05, "total": 0.95 } } ] } } CUSTOM_CSS = """ body { background-color: #030712 !important; background-image: radial-gradient(circle at 50% -20%, #1a103c, #030712 70%) !important; font-family: 'Inter', -apple-system, sans-serif !important; } .gradio-container { max-width: 1280px !important; padding: 2rem 1rem !important; } /* Glassmorphism Panels */ .glass-panel { background: rgba(17, 24, 39, 0.4) !important; backdrop-filter: blur(16px) !important; -webkit-backdrop-filter: blur(16px) !important; border: 1px solid rgba(255, 255, 255, 0.05) !important; border-radius: 20px !important; padding: 2rem !important; box-shadow: 0 12px 40px 0 rgba(0, 0, 0, 0.5) !important; } .glass-header { background: linear-gradient(135deg, rgba(139, 92, 246, 0.05), rgba(59, 130, 246, 0.05)) !important; backdrop-filter: blur(16px) !important; -webkit-backdrop-filter: blur(16px) !important; border: 1px solid rgba(255, 255, 255, 0.05) !important; border-radius: 24px !important; padding: 3rem 2rem !important; text-align: center; margin-bottom: 2.5rem; box-shadow: 0 10px 30px -10px rgba(139, 92, 246, 0.2) !important; } /* Make standard Gradio forms, boxes and groups transparent to avoid "boxes inside boxes" */ .gradio-container .form, .gradio-container .box, .gradio-container .group, .gradio-container .block, .gradio-container .panel { background: transparent !important; border: none !important; box-shadow: none !important; padding: 0 !important; } /* Title styling */ .glass-header h1 { font-family: 'Outfit', sans-serif !important; font-size: 3.5rem !important; font-weight: 900 !important; letter-spacing: -0.03em !important; background: linear-gradient(to right, #c084fc, #60a5fa) !important; -webkit-background-clip: text !important; -webkit-text-fill-color: transparent !important; margin-top: 0 !important; margin-bottom: 0.5rem !important; } .glass-header h3 { font-size: 1.25rem !important; color: #94a3b8 !important; font-weight: 500 !important; margin-top: 0 !important; margin-bottom: 0.5rem !important; } .glass-header p { font-size: 1.05rem !important; color: #cbd5e1 !important; font-style: italic !important; margin: 0 !important; } /* Tab styling overrides */ .gradio-container .tabs { border-bottom: 1px solid rgba(255, 255, 255, 0.1) !important; margin-bottom: 1.5rem !important; } .gradio-container .tabs button { border: none !important; background: transparent !important; font-size: 1.05rem !important; font-weight: 600 !important; color: #64748b !important; padding: 0.75rem 1.5rem !important; transition: all 0.3s ease !important; } .gradio-container .tabs button.selected { color: #c084fc !important; border-bottom: 2px solid #8b5cf6 !important; } /* Accent Buttons */ .accent-btn { background: linear-gradient(135deg, #6366f1, #8b5cf6) !important; color: white !important; border: none !important; font-weight: 600 !important; font-size: 1rem !important; padding: 0.75rem 1.5rem !important; border-radius: 12px !important; cursor: pointer !important; transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1) !important; } .accent-btn:hover { transform: translateY(-2px) !important; box-shadow: 0 8px 24px rgba(139, 92, 246, 0.5) !important; } .mt-8 { margin-top: 2rem !important; } /* Translucent styled sliders, dropdowns and inputs */ .gradio-container select, .gradio-container input, .gradio-container textarea, .gradio-container .input-container { background-color: rgba(10, 15, 30, 0.6) !important; border: 1px solid rgba(255, 255, 255, 0.08) !important; color: #f8fafc !important; border-radius: 10px !important; } /* Code fonts */ .code-container { font-family: 'JetBrains Mono', 'Fira Code', monospace !important; background-color: rgba(5, 8, 16, 0.8) !important; border: 1px solid rgba(255, 255, 255, 0.06) !important; border-radius: 12px !important; } /* Flatten all markdown, prose and text blocks to transparent backgrounds */ .prose, .markdown, .md, .gradio-markdown, div.prose, div.markdown, .prose *, .markdown *, .md * { background: transparent !important; border: none !important; box-shadow: none !important; } """ # ── Dynamic Leaderboard Renderer ────────────────────────────────────────────── def render_leaderboard_html(summary_data): overall = summary_data.get("overall", {}) t1 = summary_data.get("tiers", {}).get("tier1", {}) t2 = summary_data.get("tiers", {}).get("tier2", {}) t3 = summary_data.get("tiers", {}).get("tier3", {}) qwen_t1 = f"{t1.get('solve_rate', 1.0):.1%}" qwen_t2 = f"{t2.get('solve_rate', 0.774):.1%}" qwen_t3 = f"{t3.get('solve_rate', 0.381):.1%}" qwen_mean = f"{sum([t1.get('solve_rate', 1.0), t2.get('solve_rate', 0.774), t3.get('solve_rate', 0.381)]) / 3:.3f}" html = f"""
Rank Model Tier 1 (Easy) Tier 2 (Med) Tier 3 (Hard) Mean Score
🥇 1 GPT-4o 89.0% 71.0% 38.0% 0.742
🥈 2 AgentDebugger-Qwen2.5-3B-GRPO Trained {qwen_t1} {qwen_t2} {qwen_t3} {qwen_mean}
🥉 3 Llama-3.1-70B-Instruct Baseline 21.0% 21.5% 21.5% 0.210
4 Qwen2.5-Coder-3B-Instruct Base Model 15.0% 8.0% 2.0% 0.083
""" return html # ── Dynamic Trajectory Viewer Callback ──────────────────────────────────────── def get_trajectory_explorer_dropdowns(eval_data): options = [] # Load from evaluation results if available if "results" in eval_data and eval_data["results"]: for tier_name, bugs in eval_data["results"].items(): for bug in bugs: options.append(f"{bug.get('function_name')} ({tier_name.capitalize()})") # Fallback/Merge with default mock cases for name in MOCK_TRAJECTORIES.keys(): if name not in options: options.append(name) return options def get_bug_details(selected_name, eval_data): # Check mock trajectories first if selected_name in MOCK_TRAJECTORIES: data = MOCK_TRAJECTORIES[selected_name] buggy_code = data["buggy_code"] initial_error = data["initial_error"] traj = data["trajectory"] else: # Resolve from evaluation results resolved = None for tier_name, bugs in eval_data.get("results", {}).items(): for bug in bugs: if f"{bug.get('function_name')} ({tier_name.capitalize()})" == selected_name: resolved = bug break if resolved: break if resolved: buggy_code = resolved.get("prompt", "").split("```python\n")[-1].split("\n```")[0] initial_error = resolved.get("prompt", "").split("Initial failure: ")[-1].split("\n")[0] traj = [{ "turn": 1, "observation": resolved.get("raw_completion", "").split("OBSERVATION:")[1].split("HYPOTHESIS:")[0].strip(), "hypothesis": resolved.get("raw_completion", "").split("HYPOTHESIS:")[1].split("CONFIDENCE:")[0].strip(), "confidence": resolved.get("raw_completion", "").split("CONFIDENCE:")[1].split("ACTION:")[0].strip(), "action": resolved.get("raw_completion", "").split("ACTION:")[1].split("DETAIL:")[0].strip(), "detail": resolved.get("raw_completion", "").split("DETAIL:")[1].strip(), "test_results": resolved.get("test_results", {}), "reward_breakdown": resolved.get("reward_breakdown", {}) }] else: return "No code", "No error", "No trajectories available" # Format the trajectory beautifully into Markdown markdown_out = [] for step in traj: passed = step["test_results"].get("passed", 0) total = step["test_results"].get("total", 1) tests_bar = "█" * passed + "░" * (total - passed) # Color-coded action badge action_color = "#8b5cf6" if step["action"] == "propose_fix" else "#3b82f6" rb = step["reward_breakdown"] format_val = rb.get("format_compliance", rb.get("format_match", 0.0)) hypothesis_val = rb.get("hypothesis_quality", 0.0) localization_val = rb.get("localization", rb.get("syntax_correctness", 0.0)) fix_val = rb.get("fix_quality", rb.get("functionality_reward", 0.0)) semantic_val = rb.get("semantic_similarity", 0.0) markdown_out.append(f""" ### 🔄 TURN {step['turn']} --- * **🕵️ Observation:** > {step['observation']} * **💡 Hypothesis:** > {step['hypothesis']} * **🎯 Confidence:** `{step['confidence']}` * **🛠️ Action:** {step['action']} **Proposed Fix / Detail:** ```python {step['detail']} ``` **Sandbox Exec Results:** * `Tests Passed`: **{passed} / {total}** `[{tests_bar}]` * `Outcome`: **{"✅ SOLVED" if passed == total else "❌ STILL FAILING"}** **Dense Reward Breakdown:** - Format Compliance: `+{format_val:.3f}` - Hypothesis Quality: `+{hypothesis_val:.3f}` - Localization: `+{localization_val:.3f}` - Fix Quality: `+{fix_val:.3f}` - Semantic Similarity: `+{semantic_val:.3f}` - **Turn Total Reward: {sum(v for k, v in rb.items() if k != 'total'):.3f}** """) return buggy_code, initial_error, "\n\n".join(markdown_out) # ── Live sandbox execution handler ──────────────────────────────────────────── def run_sandbox_code(user_code, test_suite): # Import execution sandbox dynamically try: from env.sandbox import execute_code output, timed_out, exec_time = execute_code(user_code, test_suite) status = "⏱️ Timed Out" if timed_out else f"✓ Run in {exec_time}ms" return output, status except Exception as e: return f"Execution Error: {e}", "❌ Failed" # ── Technical Report Reader ─────────────────────────────────────────────────── def read_technical_report(): report_path = "Blog.md" if os.path.exists(report_path): try: with open(report_path, "r") as f: return f.read() except Exception: pass return "Technical report draft `Blog.md` not found." # ── Gradio App Layout ───────────────────────────────────────────────────────── eval_data = load_evaluation_data() bug_options = get_trajectory_explorer_dropdowns(eval_data) with gr.Blocks(title="AgentDebuggerEnv Research Hub") as demo: # ── Header ──────────────────────────────────────────────────────────────── gr.HTML( """

🐞 AgentDebuggerEnv

Interactive Research Showcase & Leaderboard

Aligning LLMs on Hypothesis-Driven Debugging using GRPO Reinforcement Learning

""" ) with gr.Tabs(): # ── Tab 1: Trajectory Explorer ──────────────────────────────────────── with gr.TabItem("🕵️ Trajectory Explorer"): gr.Markdown( """ ### Interactive Bug Debugging Visualizer Select a bug below to see how our fine-tuned **AgentDebugger-Qwen2.5-3B-GRPO** model localizes, hypothesizes, and patches the defect in a single step inside the sandboxed environment. """ ) with gr.Row(): with gr.Column(scale=1, elem_classes=["glass-panel"]): bug_dropdown = gr.Dropdown( choices=bug_options, value=bug_options[0] if bug_options else None, label="Choose a Curriculum Bug", interactive=True ) bug_code_viewer = gr.Code( language="python", label="Buggy Code Input", interactive=False, lines=12, elem_classes=["code-container"] ) error_msg_viewer = gr.Textbox( label="Sandbox Initial Error Output", interactive=False, lines=3 ) with gr.Column(scale=2, elem_classes=["glass-panel"]): gr.Markdown("### 🧠 Model Cognitive Loop Trajectory") trajectory_output = gr.Markdown(value="Loading initial trajectory...") # Wire up explorer update def update_explorer(name): code, err, traj = get_bug_details(name, eval_data) return code, err, traj bug_dropdown.change( fn=update_explorer, inputs=bug_dropdown, outputs=[bug_code_viewer, error_msg_viewer, trajectory_output] ) # Initial load callback demo.load( fn=lambda: update_explorer(bug_options[0]) if bug_options else ("", "", ""), outputs=[bug_code_viewer, error_msg_viewer, trajectory_output] ) # ── Tab 2: Leaderboard & Metrics ────────────────────────────────────── with gr.TabItem("📊 Benchmark Leaderboard"): gr.Markdown( """ ### Benchmark Rankings on 90 Hand-Validated Bugs We rank models based on their average score across 3 tiers of difficulty (Easy, Medium, Hard). *Scores measure formatting, hypothesis accuracy, fault localization, and test suite pass rate.* """ ) leaderboard_frame = gr.HTML(value=render_leaderboard_html(eval_data.get("summary", DEFAULT_STATS["summary"]))) with gr.Row(elem_classes=["glass-panel", "mt-8"]): with gr.Column(): gr.Markdown( """ ### 📈 Training Learning Curves (GRPO) Our reinforcement learning runs demonstrate rapid policy adaptation of Qwen-3B-Coder: - **Format compliance**: Hit 1.0 (max) within the first 50 steps. - **Total Reward**: Climbed from baseline ~0.4 to peaks of ~1.0 by step 250. - **Curriculum Transition**: Textbook drop-and-recover curve at step 150 (Tier 2 escalation). """ ) with gr.Column(): # Display metrics images from repo gr.Image("images/total.png", label="GRPO Total Reward Curve") gr.Image("images/format_compliance.png", label="Format Compliance Curve") # ── Tab 3: Sandbox Playground ───────────────────────────────────────── with gr.TabItem("🛡️ Sandbox Playground"): gr.Markdown( """ ### Hardened Sandbox Execution Environment Test arbitrary Python code against custom tests. Our execution sandbox enforces CPU limits (10s), memory limits (256MB), and blocks unsafe functions. """ ) with gr.Row(): with gr.Column(scale=1, elem_classes=["glass-panel"]): user_code = gr.Code( language="python", label="Python Code", value="def add(a, b):\n return a + b", lines=10, elem_classes=["code-container"] ) test_suite_code = gr.Code( language="python", label="Test Assertions (must print PASS or FAIL)", value="assert add(2, 3) == 5\nprint('PASS')", lines=5, elem_classes=["code-container"] ) run_btn = gr.Button("🚀 Run in Sandbox", elem_classes=["accent-btn"]) with gr.Column(scale=1, elem_classes=["glass-panel"]): sandbox_status = gr.Textbox(label="Sandbox Status", value="Ready") sandbox_stdout = gr.Code( label="Terminal Output (Stdout/Stderr)", interactive=False, lines=15, elem_classes=["code-container"] ) run_btn.click( fn=run_sandbox_code, inputs=[user_code, test_suite_code], outputs=[sandbox_stdout, sandbox_status] ) # ── Tab 4: Technical Report ─────────────────────────────────────────── with gr.TabItem("📝 Technical Report"): gr.Markdown( """ ### Research Writeup & Key Insights Read our draft paper detailing the project context, reward shaping formulations, and empirical comparisons. """ ) with gr.Group(elem_classes=["glass-panel"]): gr.Markdown(value=read_technical_report()) gr.Markdown( """ ---

Submitted to the Meta + PyTorch + Hugging Face OpenEnv Hackathon | View GitHub Repository

""" ) if __name__ == "__main__": demo.launch(server_name="0.0.0.0", server_port=7860, css=CUSTOM_CSS)