""" Incident grader — verifies that the SRE incident has been fully resolved. Three checks are performed, each worth 1/3 of the total score: 1. **Service alive** — GET /health returns HTTP 200. 2. **DB state clean** — no job rows remain with status='pending'. 3. **Code hardened** — worker.py contains a try/except block that wraps the json.loads call (verified via AST analysis, not text search). """ import ast import logging import sqlite3 import urllib.error import urllib.request from pathlib import Path from debug_env.server.schemas import ToolResult logger = logging.getLogger(__name__) # ── Public API ────────────────────────────────────────────────────────────── def grade_incident(workdir: str, service_manager, entrypoint: str) -> ToolResult: """ Run the three incident-resolution checks and return a scored ToolResult. Args: workdir: Absolute path to the task working directory. service_manager: :class:`~debug_env.server.services.ServiceManager` instance. entrypoint: Server script filename (unused here, kept for symmetry). Returns: :class:`ToolResult` where ``pass_rate = checks_passed / 3``. """ results: list[str] = [] checks_passed = 0 # ── Check 1: service liveness ────────────────────────────────────────── port = service_manager.port if service_manager is not None else None if port is not None: try: with urllib.request.urlopen( f"http://localhost:{port}/health", timeout=3 ) as resp: if resp.status == 200: checks_passed += 1 results.append("[1/3] PASS Service is healthy (GET /health → 200)") else: results.append(f"[1/3] FAIL Service returned HTTP {resp.status}") except urllib.error.URLError as exc: results.append(f"[1/3] FAIL Service unreachable: {exc.reason}") except Exception as exc: results.append(f"[1/3] FAIL Health check error: {exc}") else: results.append("[1/3] FAIL Service not started (no port assigned)") # ── Check 2: database clean (no pending jobs) ────────────────────────── db_path = Path(workdir) / "app.db" if db_path.exists(): try: conn = sqlite3.connect(str(db_path)) (pending_count,) = conn.execute( "SELECT COUNT(*) FROM jobs WHERE status='pending'" ).fetchone() conn.close() if pending_count == 0: checks_passed += 1 results.append("[2/3] PASS DB is clean — no pending jobs remain") else: results.append( f"[2/3] FAIL DB still has {pending_count} pending job(s) " "(fix or delete the poisoned row)" ) except sqlite3.Error as exc: results.append(f"[2/3] FAIL DB check error: {exc}") else: results.append("[2/3] FAIL app.db not found in workdir") # ── Check 3: worker.py hardened with try/except ──────────────────────── worker_path = Path(workdir) / "worker.py" if worker_path.exists(): try: source = worker_path.read_text(encoding="utf-8") tree = ast.parse(source) if _json_loads_inside_try(tree): checks_passed += 1 results.append( "[3/3] PASS worker.py has try/except wrapping json.loads" ) else: results.append( "[3/3] FAIL worker.py still calls json.loads outside a try/except" ) except SyntaxError as exc: results.append(f"[3/3] FAIL worker.py has a syntax error: {exc}") else: results.append("[3/3] FAIL worker.py not found in workdir") pass_rate = checks_passed / 3.0 summary = ( f"\nResult: {checks_passed}/3 checks passed " f"(pass_rate={pass_rate:.4f})" ) all_output = "\n".join(results) + summary logger.info( "grade_incident: %d/3 checks passed (pass_rate=%.4f)", checks_passed, pass_rate ) return ToolResult(pass_rate=pass_rate, logs=all_output, success=True) # ── AST helpers ───────────────────────────────────────────────────────────── def _json_loads_inside_try(tree: ast.AST) -> bool: """ Return True if any ``try`` block's body contains a ``json.loads(...)`` call. Accepts both ``json.loads(...)`` and ``loads(...)`` (from ``from json import loads``). """ for node in ast.walk(tree): if not isinstance(node, ast.Try): continue # Walk only the try body, not the except/else/finally clauses for child in _walk_nodes(node.body): if not isinstance(child, ast.Call): continue func = child.func if isinstance(func, ast.Attribute): # json.loads(...) if ( func.attr == "loads" and isinstance(func.value, ast.Name) and func.value.id == "json" ): return True elif isinstance(func, ast.Name) and func.id == "loads": # loads(...) — covers `from json import loads` return True return False def _walk_nodes(stmts: list) -> "ast.AST": """Yield all AST nodes reachable from a list of statements.""" for stmt in stmts: yield from ast.walk(stmt)