""" graders/performance.py — Relative performance grader. Weight: 10% of total reward. Never uses absolute millisecond thresholds — machines vary. Score = 1.0 means agent matches optimal speed. Score = 0.0 means agent is as slow as the naive solution. Intermediate: linear interpolation. Also checks memory via tracemalloc (peak bytes). """ from sandbox.executor import safe_exec from typing import Dict, Any def grade_performance(code: str, task: dict) -> Dict[str, Any]: """ Grade performance relative to naive and optimal baselines. Uses task['naive_baseline'] timing hints since we can't run all baselines live. For the hackathon, we use a hybrid approach: - Measure actual execution time via subprocess - Compare against task-defined naive_baseline hints - Bonus for efficient algorithms (no nested loops on large inputs) """ naive_baseline = task.get("naive_baseline", {}) naive_time_ms = naive_baseline.get("time_ms", 10) # Build a timing harness timer_code = f""" {code} import time, json, tracemalloc _test_input = {repr(task.get("perf_input", "test_input_for_perf"))} # Warmup try: run_task(_test_input) except Exception: pass # Time 3 runs tracemalloc.start() _times = [] for _ in range(3): _t0 = time.perf_counter() try: run_task(_test_input) except Exception: pass _times.append((time.perf_counter() - _t0) * 1000) _, _peak = tracemalloc.get_traced_memory() tracemalloc.stop() print(json.dumps({{ "avg_ms": sum(_times) / len(_times), "min_ms": min(_times), "peak_kb": _peak / 1024, }})) """ result = safe_exec(timer_code, "", timeout=10) if not result["ok"]: return { "score": 0.5, "feedback": "Could not measure performance — code may have errors.", } out = result.get("output", {}) if not isinstance(out, dict): return {"score": 0.5, "feedback": "Performance measurement failed."} avg_ms = out.get("avg_ms", naive_time_ms) peak_kb = out.get("peak_kb", 100) # Score relative to naive baseline # If faster than naive → >=0.5 score; if at naive speed → 0.5; faster → higher if naive_time_ms > 0: ratio = avg_ms / naive_time_ms if ratio <= 0.5: time_score = 1.0 elif ratio <= 1.0: time_score = 1.0 - 0.5 * (ratio - 0.5) / 0.5 elif ratio <= 2.0: time_score = 0.5 - 0.3 * (ratio - 1.0) else: time_score = max(0.1, 0.2 - 0.05 * (ratio - 2.0)) else: time_score = 0.7 # Memory score: penalise if using >1MB for simple tasks if peak_kb < 100: mem_score = 1.0 elif peak_kb < 500: mem_score = 0.8 elif peak_kb < 2000: mem_score = 0.6 else: mem_score = max(0.2, 1.0 - peak_kb / 10000) final = round(time_score * 0.7 + mem_score * 0.3, 4) return { "score": final, "feedback": ( f"avg={avg_ms:.1f}ms, peak_mem={peak_kb:.0f}KB. " f"Time score={time_score:.2f}, Memory score={mem_score:.2f}." ), "avg_ms": avg_ms, "peak_kb": peak_kb, }