graders/performance.py

"""
graders/performance.py — Relative performance grader.
Weight: 10% of total reward.

Never uses absolute millisecond thresholds — machines vary.
Score = 1.0 means agent matches optimal speed.
Score = 0.0 means agent is as slow as the naive solution.
Intermediate: linear interpolation.

Also checks memory via tracemalloc (peak bytes).
"""
from sandbox.executor import safe_exec
from typing import Dict, Any


def grade_performance(code: str, task: dict) -> Dict[str, Any]:
    """
    Grade performance relative to naive and optimal baselines.
    Uses task['naive_baseline'] timing hints since we can't run all baselines live.

    For the hackathon, we use a hybrid approach:
      - Measure actual execution time via subprocess
      - Compare against task-defined naive_baseline hints
      - Bonus for efficient algorithms (no nested loops on large inputs)
    """
    naive_baseline = task.get("naive_baseline", {})
    naive_time_ms = naive_baseline.get("time_ms", 10)

    # Build a timing harness
    timer_code = f"""
{code}

import time, json, tracemalloc

_test_input = {repr(task.get("perf_input", "test_input_for_perf"))}

# Warmup
try:
    run_task(_test_input)
except Exception:
    pass

# Time 3 runs
tracemalloc.start()
_times = []
for _ in range(3):
    _t0 = time.perf_counter()
    try:
        run_task(_test_input)
    except Exception:
        pass
    _times.append((time.perf_counter() - _t0) * 1000)

_, _peak = tracemalloc.get_traced_memory()
tracemalloc.stop()

print(json.dumps({{
    "avg_ms": sum(_times) / len(_times),
    "min_ms": min(_times),
    "peak_kb": _peak / 1024,
}}))
"""
    result = safe_exec(timer_code, "", timeout=10)

    if not result["ok"]:
        return {
            "score": 0.5,
            "feedback": "Could not measure performance — code may have errors.",
        }

    out = result.get("output", {})
    if not isinstance(out, dict):
        return {"score": 0.5, "feedback": "Performance measurement failed."}

    avg_ms = out.get("avg_ms", naive_time_ms)
    peak_kb = out.get("peak_kb", 100)

    # Score relative to naive baseline
    # If faster than naive → >=0.5 score; if at naive speed → 0.5; faster → higher
    if naive_time_ms > 0:
        ratio = avg_ms / naive_time_ms
        if ratio <= 0.5:
            time_score = 1.0
        elif ratio <= 1.0:
            time_score = 1.0 - 0.5 * (ratio - 0.5) / 0.5
        elif ratio <= 2.0:
            time_score = 0.5 - 0.3 * (ratio - 1.0)
        else:
            time_score = max(0.1, 0.2 - 0.05 * (ratio - 2.0))
    else:
        time_score = 0.7

    # Memory score: penalise if using >1MB for simple tasks
    if peak_kb < 100:
        mem_score = 1.0
    elif peak_kb < 500:
        mem_score = 0.8
    elif peak_kb < 2000:
        mem_score = 0.6
    else:
        mem_score = max(0.2, 1.0 - peak_kb / 10000)

    final = round(time_score * 0.7 + mem_score * 0.3, 4)

    return {
        "score": final,
        "feedback": (
            f"avg={avg_ms:.1f}ms, peak_mem={peak_kb:.0f}KB. "
            f"Time score={time_score:.2f}, Memory score={mem_score:.2f}."
        ),
        "avg_ms": avg_ms,
        "peak_kb": peak_kb,
    }