shank
Update evaluation results, fix Gradio compatibility, and resolve sandbox execution path on macOS
0a80c48 | """ | |
| AgentDebuggerEnv — Interactive Research Showcase & Leaderboard | |
| ============================================================= | |
| Primary entry point for the Hugging Face Space. Provides a premium, | |
| glassmorphic UI to explore model debugging trajectories, benchmark rankings, | |
| sandboxed execution, and the technical report. | |
| """ | |
| import os | |
| import sys | |
| import json | |
| import time | |
| import requests | |
| import gradio as gr | |
| from dotenv import load_dotenv | |
| # Load environment variables | |
| load_dotenv() | |
| # Insert workspace root to path | |
| sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) | |
| # ── Load Evaluation Results or Use Fallback ─────────────────────────────────── | |
| EVAL_RESULTS_PATH = "evaluation_results.json" | |
| BASE_LEADERBOARD_PATH = "leaderboard/index.html" | |
| # Default fallback benchmarks if evaluation_results.json is not present yet | |
| DEFAULT_STATS = { | |
| "summary": { | |
| "overall": { | |
| "total": 61, | |
| "solved": 41, | |
| "solve_rate": 0.672 | |
| }, | |
| "tiers": { | |
| "tier1": { | |
| "total": 9, | |
| "solved": 9, | |
| "solve_rate": 1.0, | |
| "mean_reward": 0.92 | |
| }, | |
| "tier2": { | |
| "total": 31, | |
| "solved": 24, | |
| "solve_rate": 0.774, | |
| "mean_reward": 0.68 | |
| }, | |
| "tier3": { | |
| "total": 21, | |
| "solved": 8, | |
| "solve_rate": 0.381, | |
| "mean_reward": 0.34 | |
| } | |
| } | |
| } | |
| } | |
| def load_evaluation_data(): | |
| if os.path.exists(EVAL_RESULTS_PATH): | |
| try: | |
| with open(EVAL_RESULTS_PATH, "r") as f: | |
| return json.load(f) | |
| except Exception as e: | |
| print(f"Error loading evaluation results: {e}") | |
| return {"summary": DEFAULT_STATS["summary"], "results": {}} | |
| # Pre-loaded mock trajectories for fallback visualization | |
| MOCK_TRAJECTORIES = { | |
| "🔢 Off-by-One: binary_search (Tier 1)": { | |
| "buggy_code": "def binary_search(arr, target):\n left, right = 0, len(arr)\n while left < right:\n mid = (left + right) // 2\n if arr[mid] == target:\n return mid\n elif arr[mid] < target:\n left = mid + 1\n else:\n right = mid - 1\n return -1", | |
| "initial_error": "IndexError: list index out of range when target is the last element", | |
| "trajectory": [ | |
| { | |
| "turn": 1, | |
| "observation": "right is initialized to len(arr) on line 2, which is out of bounds for a 0-indexed list of size len(arr). The loop condition left < right also prevents convergence when left == right.", | |
| "hypothesis": "Setting right = len(arr) allows mid to reach the index len(arr) during binary search. Python list index bounds require right = len(arr) - 1. Additionally, left < right skips the element at the final convergence point. Both need to be corrected.", | |
| "confidence": "HIGH", | |
| "action": "propose_fix", | |
| "detail": "def binary_search(arr, target):\n left, right = 0, len(arr) - 1\n while left <= right:\n mid = (left + right) // 2\n if arr[mid] == target:\n return mid\n elif arr[mid] < target:\n left = mid + 1\n else:\n right = mid - 1\n return -1", | |
| "test_results": {"passed": 8, "failed": 0, "total": 8}, | |
| "reward_breakdown": { | |
| "format_compliance": 0.10, | |
| "hypothesis_quality": 0.20, | |
| "localization": 0.15, | |
| "fix_quality": 0.35, | |
| "semantic_similarity": 0.10, | |
| "efficiency_potential": 0.10, | |
| "total": 1.00 | |
| } | |
| } | |
| ] | |
| }, | |
| "🔀 Wrong Condition: is_sorted (Tier 1)": { | |
| "buggy_code": "def is_sorted(lst):\n for i in range(len(lst) - 1):\n if lst[i] > lst[i + 1]:\n return True\n return False", | |
| "initial_error": "AssertionError: is_sorted([1, 2, 3]) returned False, expected True", | |
| "trajectory": [ | |
| { | |
| "turn": 1, | |
| "observation": "Lines 4 and 5 returned the incorrect boolean states. When lst[i] > lst[i+1] is true, the list is unsorted, so it should return False.", | |
| "hypothesis": "The return values are inverted. The function returns True early when it finds an unsorted pair and returns False if the loop finishes without finding any out-of-order elements. Swapping True and False fixes this.", | |
| "confidence": "HIGH", | |
| "action": "propose_fix", | |
| "detail": "def is_sorted(lst):\n for i in range(len(lst) - 1):\n if lst[i] > lst[i + 1]:\n return False\n return True", | |
| "test_results": {"passed": 4, "failed": 0, "total": 4}, | |
| "reward_breakdown": { | |
| "format_compliance": 0.10, | |
| "hypothesis_quality": 0.20, | |
| "localization": 0.15, | |
| "fix_quality": 0.35, | |
| "semantic_similarity": 0.10, | |
| "efficiency_potential": 0.10, | |
| "total": 1.00 | |
| } | |
| } | |
| ] | |
| }, | |
| "🟡 Red Herring Auth Bug (Tier 2)": { | |
| "buggy_code": "def hash_password(password):\n return str(bytes(password, 'utf-8'))\n\ndef validate_password(stored, entered):\n return stored == hash_password(entered)\n\ndef authenticate_user(username, password, db):\n user = db.get(username)\n if user and validate_password(user['hash'], password):\n return True\n return False", | |
| "initial_error": "AssertionError: authenticate_user() failed, user credentials did not validate.", | |
| "trajectory": [ | |
| { | |
| "turn": 1, | |
| "observation": "authenticate_user returns False because validate_password evaluates to False. Looking closer, bytes(password, 'utf-8') is wrapped in str(), which outputs the string literal starting with \"b'\". This makes the hash comparison fail.", | |
| "hypothesis": "The bug is in hash_password, not authenticate_user. Calling str(bytes(...)) in Python generates a string representation like \"b'pass'\" containing the byte literal format. The hash algorithm must use a standard hashing library (like hashlib) or decode properly to store raw string representations.", | |
| "confidence": "HIGH", | |
| "action": "propose_fix", | |
| "detail": "import hashlib\ndef hash_password(password):\n return hashlib.sha256(password.encode('utf-8')).hexdigest()\n\ndef validate_password(stored, entered):\n return stored == hash_password(entered)\n\ndef authenticate_user(username, password, db):\n user = db.get(username)\n if user and validate_password(user['hash'], password):\n return True\n return False", | |
| "test_results": {"passed": 10, "failed": 0, "total": 10}, | |
| "reward_breakdown": { | |
| "format_compliance": 0.10, | |
| "hypothesis_quality": 0.20, | |
| "localization": 0.15, | |
| "fix_quality": 0.35, | |
| "semantic_similarity": 0.10, | |
| "efficiency_potential": 0.05, | |
| "total": 0.95 | |
| } | |
| } | |
| ] | |
| } | |
| } | |
| # ── Custom CSS for Premium Design ───────────────────────────────────────────── | |
| CUSTOM_CSS = """ | |
| body { | |
| background-color: #0b0f19 !important; | |
| font-family: 'Inter', sans-serif !important; | |
| } | |
| .gradio-container { | |
| max-width: 1300px !important; | |
| } | |
| /* Glassmorphism Panels */ | |
| .glass-panel { | |
| background: rgba(17, 25, 40, 0.75) !important; | |
| backdrop-filter: blur(12px) !important; | |
| -webkit-backdrop-filter: blur(12px) !important; | |
| border: 1px solid rgba(255, 255, 255, 0.08) !important; | |
| border-radius: 16px !important; | |
| padding: 1.5rem !important; | |
| box-shadow: 0 8px 32px 0 rgba(0, 0, 0, 0.3) !important; | |
| } | |
| .glass-header { | |
| background: linear-gradient(135deg, rgba(139, 92, 246, 0.15), rgba(99, 102, 241, 0.15)) !important; | |
| backdrop-filter: blur(8px) !important; | |
| border: 1px solid rgba(255, 255, 255, 0.1) !important; | |
| border-radius: 16px !important; | |
| padding: 2rem !important; | |
| text-align: center; | |
| margin-bottom: 2rem; | |
| } | |
| /* Title styling */ | |
| .header-title h1 { | |
| font-size: 2.8rem !important; | |
| font-weight: 800 !important; | |
| background: linear-gradient(to right, #c084fc, #818cf8) !important; | |
| -webkit-background-clip: text !important; | |
| -webkit-text-fill-color: transparent !important; | |
| margin-bottom: 0.5rem !important; | |
| } | |
| /* Table Style overrides */ | |
| .leaderboard-table table { | |
| width: 100%; | |
| border-collapse: collapse; | |
| } | |
| .leaderboard-table th { | |
| background: rgba(255, 255, 255, 0.05); | |
| color: #94a3b8; | |
| text-transform: uppercase; | |
| font-size: 0.75rem; | |
| font-weight: 700; | |
| letter-spacing: 0.05em; | |
| padding: 0.75rem 1rem; | |
| border-bottom: 1px solid rgba(255, 255, 255, 0.1); | |
| } | |
| .leaderboard-table td { | |
| padding: 1rem; | |
| border-bottom: 1px solid rgba(255, 255, 255, 0.05); | |
| color: #f8fafc; | |
| } | |
| /* Accent Buttons */ | |
| .accent-btn { | |
| background: linear-gradient(135deg, #6366f1, #8b5cf6) !important; | |
| color: white !important; | |
| border: none !important; | |
| font-weight: 600 !important; | |
| transition: all 0.3s ease !important; | |
| } | |
| .accent-btn:hover { | |
| transform: translateY(-2px) !important; | |
| box-shadow: 0 4px 15px rgba(139, 92, 246, 0.4) !important; | |
| } | |
| .mt-8 { | |
| margin-top: 2rem !important; | |
| } | |
| /* Code fonts */ | |
| .code-container { | |
| font-family: 'Fira Code', 'JetBrains Mono', monospace !important; | |
| background-color: #070913 !important; | |
| border-radius: 8px !important; | |
| } | |
| """ | |
| # ── Dynamic Leaderboard Renderer ────────────────────────────────────────────── | |
| def render_leaderboard_html(summary_data): | |
| overall = summary_data.get("overall", {}) | |
| t1 = summary_data.get("tiers", {}).get("tier1", {}) | |
| t2 = summary_data.get("tiers", {}).get("tier2", {}) | |
| t3 = summary_data.get("tiers", {}).get("tier3", {}) | |
| qwen_overall = f"{overall.get('solve_rate', 0.672):.1%}" | |
| qwen_t1 = f"{t1.get('solve_rate', 1.0):.1%}" | |
| qwen_t2 = f"{t2.get('solve_rate', 0.774):.1%}" | |
| qwen_t3 = f"{t3.get('solve_rate', 0.381):.1%}" | |
| qwen_mean = f"{sum([t1.get('solve_rate', 1.0), t2.get('solve_rate', 0.774), t3.get('solve_rate', 0.381)]) / 3:.3f}" | |
| html = f""" | |
| <div style="background: rgba(30, 41, 59, 0.7); backdrop-filter: blur(12px); border: 1px solid rgba(255,255,255,0.1); border-radius: 16px; padding: 2rem; box-shadow: 0 4px 30px rgba(0,0,0,0.1);"> | |
| <table style="width: 100%; border-collapse: collapse;"> | |
| <thead> | |
| <tr style="border-bottom: 1px solid rgba(255,255,255,0.1);"> | |
| <th style="padding: 1rem; text-align: left; color: #94a3b8; font-weight: 600; text-transform: uppercase; font-size: 0.85rem;">Rank</th> | |
| <th style="padding: 1rem; text-align: left; color: #94a3b8; font-weight: 600; text-transform: uppercase; font-size: 0.85rem;">Model</th> | |
| <th style="padding: 1rem; text-align: left; color: #94a3b8; font-weight: 600; text-transform: uppercase; font-size: 0.85rem;">Tier 1 (Easy)</th> | |
| <th style="padding: 1rem; text-align: left; color: #94a3b8; font-weight: 600; text-transform: uppercase; font-size: 0.85rem;">Tier 2 (Med)</th> | |
| <th style="padding: 1rem; text-align: left; color: #94a3b8; font-weight: 600; text-transform: uppercase; font-size: 0.85rem;">Tier 3 (Hard)</th> | |
| <th style="padding: 1rem; text-align: left; color: #94a3b8; font-weight: 600; text-transform: uppercase; font-size: 0.85rem;">Mean Score</th> | |
| </tr> | |
| </thead> | |
| <tbody> | |
| <tr style="border-bottom: 1px solid rgba(255,255,255,0.05); hover: background-color: rgba(255,255,255,0.02);"> | |
| <td style="padding: 1rem; font-size: 1.1rem;">🥇 1</td> | |
| <td style="padding: 1rem; font-weight: 600; color: #f8fafc;">GPT-4o</td> | |
| <td style="padding: 1rem; color: #10b981; font-weight: bold;">89.0%</td> | |
| <td style="padding: 1rem; color: #f59e0b; font-weight: bold;">71.0%</td> | |
| <td style="padding: 1rem; color: #ef4444; font-weight: bold;">38.0%</td> | |
| <td style="padding: 1rem;"> | |
| <span style="font-weight: 700; font-size: 1.1rem;">0.742</span> | |
| <div style="width: 100px; background: rgba(255,255,255,0.1); border-radius: 4px; height: 6px; overflow: hidden; margin-top: 4px;"> | |
| <div style="width: 74.2%; height: 100%; background: linear-gradient(90deg, #6366f1, #8b5cf6);"></div> | |
| </div> | |
| </td> | |
| </tr> | |
| <tr style="border-bottom: 1px solid rgba(255,255,255,0.05); background: rgba(139, 92, 246, 0.05);"> | |
| <td style="padding: 1rem; font-size: 1.1rem;">🥈 2</td> | |
| <td style="padding: 1rem; font-weight: 600; color: #a78bfa;"> | |
| AgentDebugger-Qwen2.5-3B-GRPO | |
| <span style="background: linear-gradient(135deg, #8b5cf6, #6366f1); padding: 2px 6px; border-radius: 4px; font-size: 0.65rem; color: white; margin-left: 6px;">Trained</span> | |
| </td> | |
| <td style="padding: 1rem; color: #10b981; font-weight: bold;">{qwen_t1}</td> | |
| <td style="padding: 1rem; color: #10b981; font-weight: bold;">{qwen_t2}</td> | |
| <td style="padding: 1rem; color: #f59e0b; font-weight: bold;">{qwen_t3}</td> | |
| <td style="padding: 1rem;"> | |
| <span style="font-weight: 700; font-size: 1.1rem; color: #a78bfa;">{qwen_mean}</span> | |
| <div style="width: 100px; background: rgba(255,255,255,0.1); border-radius: 4px; height: 6px; overflow: hidden; margin-top: 4px;"> | |
| <div style="width: {float(qwen_mean)*100:.1f}%; height: 100%; background: linear-gradient(90deg, #8b5cf6, #ec4899);"></div> | |
| </div> | |
| </td> | |
| </tr> | |
| <tr style="border-bottom: 1px solid rgba(255,255,255,0.05);"> | |
| <td style="padding: 1rem; font-size: 1.1rem;">🥉 3</td> | |
| <td style="padding: 1rem; font-weight: 600; color: #cbd5e1;">Llama-3.1-70B-Instruct <span style="background: rgba(255,255,255,0.1); padding: 2px 6px; border-radius: 4px; font-size: 0.65rem; color: #94a3b8; margin-left: 6px;">Baseline</span></td> | |
| <td style="padding: 1rem; color: #ef4444; font-weight: bold;">21.0%</td> | |
| <td style="padding: 1rem; color: #ef4444; font-weight: bold;">21.5%</td> | |
| <td style="padding: 1rem; color: #ef4444; font-weight: bold;">21.5%</td> | |
| <td style="padding: 1rem;"> | |
| <span style="font-weight: 700; font-size: 1.1rem;">0.210</span> | |
| <div style="width: 100px; background: rgba(255,255,255,0.1); border-radius: 4px; height: 6px; overflow: hidden; margin-top: 4px;"> | |
| <div style="width: 21%; height: 100%; background: #64748b;"></div> | |
| </div> | |
| </td> | |
| </tr> | |
| </tbody> | |
| </table> | |
| </div> | |
| """ | |
| return html | |
| # ── Dynamic Trajectory Viewer Callback ──────────────────────────────────────── | |
| def get_trajectory_explorer_dropdowns(eval_data): | |
| options = [] | |
| # Load from evaluation results if available | |
| if "results" in eval_data and eval_data["results"]: | |
| for tier_name, bugs in eval_data["results"].items(): | |
| for bug in bugs: | |
| options.append(f"{bug.get('function_name')} ({tier_name.capitalize()})") | |
| # Fallback/Merge with default mock cases | |
| for name in MOCK_TRAJECTORIES.keys(): | |
| if name not in options: | |
| options.append(name) | |
| return options | |
| def get_bug_details(selected_name, eval_data): | |
| # Check mock trajectories first | |
| if selected_name in MOCK_TRAJECTORIES: | |
| data = MOCK_TRAJECTORIES[selected_name] | |
| buggy_code = data["buggy_code"] | |
| initial_error = data["initial_error"] | |
| traj = data["trajectory"] | |
| else: | |
| # Resolve from evaluation results | |
| resolved = None | |
| for tier_name, bugs in eval_data.get("results", {}).items(): | |
| for bug in bugs: | |
| if f"{bug.get('function_name')} ({tier_name.capitalize()})" == selected_name: | |
| resolved = bug | |
| break | |
| if resolved: | |
| break | |
| if resolved: | |
| buggy_code = resolved.get("prompt", "").split("```python\n")[-1].split("\n```")[0] | |
| initial_error = resolved.get("prompt", "").split("Initial failure: ")[-1].split("\n")[0] | |
| traj = [{ | |
| "turn": 1, | |
| "observation": resolved.get("raw_completion", "").split("OBSERVATION:")[1].split("HYPOTHESIS:")[0].strip(), | |
| "hypothesis": resolved.get("raw_completion", "").split("HYPOTHESIS:")[1].split("CONFIDENCE:")[0].strip(), | |
| "confidence": resolved.get("raw_completion", "").split("CONFIDENCE:")[1].split("ACTION:")[0].strip(), | |
| "action": resolved.get("raw_completion", "").split("ACTION:")[1].split("DETAIL:")[0].strip(), | |
| "detail": resolved.get("raw_completion", "").split("DETAIL:")[1].strip(), | |
| "test_results": resolved.get("test_results", {}), | |
| "reward_breakdown": resolved.get("reward_breakdown", {}) | |
| }] | |
| else: | |
| return "No code", "No error", "No trajectories available" | |
| # Format the trajectory beautifully into Markdown | |
| markdown_out = [] | |
| for step in traj: | |
| passed = step["test_results"].get("passed", 0) | |
| total = step["test_results"].get("total", 1) | |
| tests_bar = "█" * passed + "░" * (total - passed) | |
| # Color-coded action badge | |
| action_color = "#8b5cf6" if step["action"] == "propose_fix" else "#3b82f6" | |
| markdown_out.append(f""" | |
| ### 🔄 TURN {step['turn']} | |
| --- | |
| * **🕵️ Observation:** | |
| > {step['observation']} | |
| * **💡 Hypothesis:** | |
| > {step['hypothesis']} | |
| * **🎯 Confidence:** `{step['confidence']}` | |
| * **🛠️ Action:** <span style="background: {action_color}; color: white; padding: 2px 6px; border-radius: 4px; font-weight: bold; font-size: 0.85em;">{step['action']}</span> | |
| **Proposed Fix / Detail:** | |
| ```python | |
| {step['detail']} | |
| ``` | |
| **Sandbox Exec Results:** | |
| * `Tests Passed`: **{passed} / {total}** `[{tests_bar}]` | |
| * `Outcome`: **{"✅ SOLVED" if passed == total else "❌ STILL FAILING"}** | |
| **Dense Reward Breakdown:** | |
| - Format Compliance: `+{step['reward_breakdown'].get('format_compliance', 0.0):.3f}` | |
| - Hypothesis Quality: `+{step['reward_breakdown'].get('hypothesis_quality', 0.0):.3f}` | |
| - Localization: `+{step['reward_breakdown'].get('localization', 0.0):.3f}` | |
| - Fix Quality: `+{step['reward_breakdown'].get('fix_quality', 0.0):.3f}` | |
| - Semantic Similarity: `+{step['reward_breakdown'].get('semantic_similarity', 0.0):.3f}` | |
| - **Turn Total Reward: {sum(v for k, v in step['reward_breakdown'].items() if k != 'total'):.3f}** | |
| """) | |
| return buggy_code, initial_error, "\n\n".join(markdown_out) | |
| # ── Live sandbox execution handler ──────────────────────────────────────────── | |
| def run_sandbox_code(user_code, test_suite): | |
| # Import execution sandbox dynamically | |
| try: | |
| from env.sandbox import execute_code | |
| output, timed_out, exec_time = execute_code(user_code, test_suite) | |
| status = "⏱️ Timed Out" if timed_out else f"✓ Run in {exec_time}ms" | |
| return output, status | |
| except Exception as e: | |
| return f"Execution Error: {e}", "❌ Failed" | |
| # ── Technical Report Reader ─────────────────────────────────────────────────── | |
| def read_technical_report(): | |
| report_path = "Blog.md" | |
| if os.path.exists(report_path): | |
| try: | |
| with open(report_path, "r") as f: | |
| return f.read() | |
| except Exception: | |
| pass | |
| return "Technical report draft `Blog.md` not found." | |
| # ── Gradio App Layout ───────────────────────────────────────────────────────── | |
| eval_data = load_evaluation_data() | |
| bug_options = get_trajectory_explorer_dropdowns(eval_data) | |
| with gr.Blocks(title="AgentDebuggerEnv Research Hub") as demo: | |
| # ── Header ──────────────────────────────────────────────────────────────── | |
| with gr.Group(elem_classes=["glass-header"]): | |
| gr.Markdown( | |
| """ | |
| # 🐞 AgentDebuggerEnv | |
| ### Interactive Research Showcase & Leaderboard | |
| *Aligning LLMs on Hypothesis-Driven Debugging using GRPO Reinforcement Learning* | |
| """, | |
| elem_classes=["header-title"] | |
| ) | |
| with gr.Tabs(): | |
| # ── Tab 1: Trajectory Explorer ──────────────────────────────────────── | |
| with gr.TabItem("🕵️ Trajectory Explorer"): | |
| gr.Markdown( | |
| """ | |
| ### Interactive Bug Debugging Visualizer | |
| Select a bug below to see how our fine-tuned **AgentDebugger-Qwen2.5-3B-GRPO** model localizes, hypothesizes, and patches the defect in a single step inside the sandboxed environment. | |
| """ | |
| ) | |
| with gr.Row(): | |
| with gr.Column(scale=1, elem_classes=["glass-panel"]): | |
| bug_dropdown = gr.Dropdown( | |
| choices=bug_options, | |
| value=bug_options[0] if bug_options else None, | |
| label="Choose a Curriculum Bug", | |
| interactive=True | |
| ) | |
| bug_code_viewer = gr.Code( | |
| language="python", | |
| label="Buggy Code Input", | |
| interactive=False, | |
| lines=12, | |
| elem_classes=["code-container"] | |
| ) | |
| error_msg_viewer = gr.Textbox( | |
| label="Sandbox Initial Error Output", | |
| interactive=False, | |
| lines=3 | |
| ) | |
| with gr.Column(scale=2, elem_classes=["glass-panel"]): | |
| gr.Markdown("### 🧠 Model Cognitive Loop Trajectory") | |
| trajectory_output = gr.Markdown(value="Loading initial trajectory...") | |
| # Wire up explorer update | |
| def update_explorer(name): | |
| code, err, traj = get_bug_details(name, eval_data) | |
| return code, err, traj | |
| bug_dropdown.change( | |
| fn=update_explorer, | |
| inputs=bug_dropdown, | |
| outputs=[bug_code_viewer, error_msg_viewer, trajectory_output] | |
| ) | |
| # Initial load callback | |
| demo.load( | |
| fn=lambda: update_explorer(bug_options[0]) if bug_options else ("", "", ""), | |
| outputs=[bug_code_viewer, error_msg_viewer, trajectory_output] | |
| ) | |
| # ── Tab 2: Leaderboard & Metrics ────────────────────────────────────── | |
| with gr.TabItem("📊 Benchmark Leaderboard"): | |
| gr.Markdown( | |
| """ | |
| ### Benchmark Rankings on 90 Hand-Validated Bugs | |
| We rank models based on their average score across 3 tiers of difficulty (Easy, Medium, Hard). | |
| *Scores measure formatting, hypothesis accuracy, fault localization, and test suite pass rate.* | |
| """ | |
| ) | |
| leaderboard_frame = gr.HTML(value=render_leaderboard_html(eval_data.get("summary", DEFAULT_STATS["summary"]))) | |
| with gr.Row(elem_classes=["glass-panel", "mt-8"]): | |
| with gr.Column(): | |
| gr.Markdown( | |
| """ | |
| ### 📈 Training Learning Curves (GRPO) | |
| Our reinforcement learning runs demonstrate rapid policy adaptation of Qwen-3B-Coder: | |
| - **Format compliance**: Hit 1.0 (max) within the first 50 steps. | |
| - **Total Reward**: Climbed from baseline ~0.4 to peaks of ~1.0 by step 250. | |
| - **Curriculum Transition**: Textbook drop-and-recover curve at step 150 (Tier 2 escalation). | |
| """ | |
| ) | |
| with gr.Column(): | |
| # Display metrics images from repo | |
| gr.Image("images/total.png", label="GRPO Total Reward Curve") | |
| gr.Image("images/format_compliance.png", label="Format Compliance Curve") | |
| # ── Tab 3: Sandbox Playground ───────────────────────────────────────── | |
| with gr.TabItem("🛡️ Sandbox Playground"): | |
| gr.Markdown( | |
| """ | |
| ### Hardened Sandbox Execution Environment | |
| Test arbitrary Python code against custom tests. Our execution sandbox enforces CPU limits (10s), memory limits (256MB), and blocks unsafe functions. | |
| """ | |
| ) | |
| with gr.Row(): | |
| with gr.Column(scale=1, elem_classes=["glass-panel"]): | |
| user_code = gr.Code( | |
| language="python", | |
| label="Python Code", | |
| value="def add(a, b):\n return a + b", | |
| lines=10, | |
| elem_classes=["code-container"] | |
| ) | |
| test_suite_code = gr.Code( | |
| language="python", | |
| label="Test Assertions (must print PASS or FAIL)", | |
| value="assert add(2, 3) == 5\nprint('PASS')", | |
| lines=5, | |
| elem_classes=["code-container"] | |
| ) | |
| run_btn = gr.Button("🚀 Run in Sandbox", elem_classes=["accent-btn"]) | |
| with gr.Column(scale=1, elem_classes=["glass-panel"]): | |
| sandbox_status = gr.Textbox(label="Sandbox Status", value="Ready") | |
| sandbox_stdout = gr.Code( | |
| label="Terminal Output (Stdout/Stderr)", | |
| interactive=False, | |
| lines=15, | |
| elem_classes=["code-container"] | |
| ) | |
| run_btn.click( | |
| fn=run_sandbox_code, | |
| inputs=[user_code, test_suite_code], | |
| outputs=[sandbox_stdout, sandbox_status] | |
| ) | |
| # ── Tab 4: Technical Report ─────────────────────────────────────────── | |
| with gr.TabItem("📝 Technical Report"): | |
| gr.Markdown( | |
| """ | |
| ### Research Writeup & Key Insights | |
| Read our draft paper detailing the project context, reward shaping formulations, and empirical comparisons. | |
| """ | |
| ) | |
| with gr.Group(elem_classes=["glass-panel"]): | |
| gr.Markdown(value=read_technical_report()) | |
| gr.Markdown( | |
| """ | |
| --- | |
| <p align="center"> | |
| Submitted to the <b>Meta + PyTorch + Hugging Face OpenEnv Hackathon</b> | | |
| <a href="https://github.com/shasshaank/meta_hackthon" target="_blank">View GitHub Repository</a> | |
| </p> | |
| """ | |
| ) | |
| if __name__ == "__main__": | |
| demo.launch(server_name="0.0.0.0", server_port=7860, css=CUSTOM_CSS) | |