debug_env/server/graders/incident_grader.py

"""
Incident grader — verifies that the SRE incident has been fully resolved.

Three checks are performed, each worth 1/3 of the total score:

1. **Service alive** — GET /health returns HTTP 200.
2. **DB state clean** — no job rows remain with status='pending'.
3. **Code hardened** — worker.py contains a try/except block that wraps
   the json.loads call (verified via AST analysis, not text search).
"""

import ast
import logging
import sqlite3
import urllib.error
import urllib.request
from pathlib import Path

from debug_env.server.schemas import ToolResult

logger = logging.getLogger(__name__)


# ── Public API ──────────────────────────────────────────────────────────────


def grade_incident(workdir: str, service_manager, entrypoint: str) -> ToolResult:
    """
    Run the three incident-resolution checks and return a scored ToolResult.

    Args:
        workdir:         Absolute path to the task working directory.
        service_manager: :class:`~debug_env.server.services.ServiceManager` instance.
        entrypoint:      Server script filename (unused here, kept for symmetry).

    Returns:
        :class:`ToolResult` where ``pass_rate = checks_passed / 3``.
    """
    results: list[str] = []
    checks_passed = 0

    # ── Check 1: service liveness ──────────────────────────────────────────
    port = service_manager.port if service_manager is not None else None
    if port is not None:
        try:
            with urllib.request.urlopen(
                f"http://localhost:{port}/health", timeout=3
            ) as resp:
                if resp.status == 200:
                    checks_passed += 1
                    results.append("[1/3] PASS  Service is healthy (GET /health → 200)")
                else:
                    results.append(f"[1/3] FAIL  Service returned HTTP {resp.status}")
        except urllib.error.URLError as exc:
            results.append(f"[1/3] FAIL  Service unreachable: {exc.reason}")
        except Exception as exc:
            results.append(f"[1/3] FAIL  Health check error: {exc}")
    else:
        results.append("[1/3] FAIL  Service not started (no port assigned)")

    # ── Check 2: database clean (no pending jobs) ──────────────────────────
    db_path = Path(workdir) / "app.db"
    if db_path.exists():
        try:
            conn = sqlite3.connect(str(db_path))
            (pending_count,) = conn.execute(
                "SELECT COUNT(*) FROM jobs WHERE status='pending'"
            ).fetchone()
            conn.close()
            if pending_count == 0:
                checks_passed += 1
                results.append("[2/3] PASS  DB is clean — no pending jobs remain")
            else:
                results.append(
                    f"[2/3] FAIL  DB still has {pending_count} pending job(s) "
                    "(fix or delete the poisoned row)"
                )
        except sqlite3.Error as exc:
            results.append(f"[2/3] FAIL  DB check error: {exc}")
    else:
        results.append("[2/3] FAIL  app.db not found in workdir")

    # ── Check 3: worker.py hardened with try/except ────────────────────────
    worker_path = Path(workdir) / "worker.py"
    if worker_path.exists():
        try:
            source = worker_path.read_text(encoding="utf-8")
            tree = ast.parse(source)
            if _json_loads_inside_try(tree):
                checks_passed += 1
                results.append(
                    "[3/3] PASS  worker.py has try/except wrapping json.loads"
                )
            else:
                results.append(
                    "[3/3] FAIL  worker.py still calls json.loads outside a try/except"
                )
        except SyntaxError as exc:
            results.append(f"[3/3] FAIL  worker.py has a syntax error: {exc}")
    else:
        results.append("[3/3] FAIL  worker.py not found in workdir")

    pass_rate = checks_passed / 3.0
    summary = (
        f"\nResult: {checks_passed}/3 checks passed  "
        f"(pass_rate={pass_rate:.4f})"
    )
    all_output = "\n".join(results) + summary

    logger.info(
        "grade_incident: %d/3 checks passed (pass_rate=%.4f)", checks_passed, pass_rate
    )
    return ToolResult(pass_rate=pass_rate, logs=all_output, success=True)


# ── AST helpers ─────────────────────────────────────────────────────────────


def _json_loads_inside_try(tree: ast.AST) -> bool:
    """
    Return True if any ``try`` block's body contains a ``json.loads(...)`` call.

    Accepts both ``json.loads(...)`` and ``loads(...)`` (from ``from json import loads``).
    """
    for node in ast.walk(tree):
        if not isinstance(node, ast.Try):
            continue
        # Walk only the try body, not the except/else/finally clauses
        for child in _walk_nodes(node.body):
            if not isinstance(child, ast.Call):
                continue
            func = child.func
            if isinstance(func, ast.Attribute):
                # json.loads(...)
                if (
                    func.attr == "loads"
                    and isinstance(func.value, ast.Name)
                    and func.value.id == "json"
                ):
                    return True
            elif isinstance(func, ast.Name) and func.id == "loads":
                # loads(...) — covers `from json import loads`
                return True
    return False


def _walk_nodes(stmts: list) -> "ast.AST":
    """Yield all AST nodes reachable from a list of statements."""
    for stmt in stmts:
        yield from ast.walk(stmt)