Spaces:
Sleeping
Sleeping
| #!/usr/bin/env python3 | |
| """Pre-submission validation script for CodeOrganismVM.""" | |
| from __future__ import annotations | |
| import argparse | |
| import os | |
| import sys | |
| from typing import Callable | |
| import requests | |
| TASK_IDS = ["phase_1", "phase_2", "phase_3"] | |
| PASS = "\033[92mPASS\033[0m" | |
| FAIL = "\033[91mFAIL\033[0m" | |
| API_HEADERS: dict[str, str] = {} | |
| def check(name: str, condition: bool, detail: str = "") -> bool: | |
| status = PASS if condition else FAIL | |
| msg = f" [{status}] {name}" | |
| if detail: | |
| msg += f" — {detail}" | |
| print(msg) | |
| return condition | |
| def _check_health(url: str, tally: Callable[[bool], None]) -> None: | |
| print("[1/7] Health Check") | |
| try: | |
| response = requests.get(f"{url}/", timeout=10) | |
| tally(check("GET / returns 200", response.status_code == 200)) | |
| body = response.json() | |
| tally(check("Response has 'status' field", "status" in body, str(body))) | |
| tally(check("Environment is autonomous-sre", body.get("environment") == "autonomous-sre")) | |
| except requests.RequestException as exc: | |
| tally(check("Server reachable", False, str(exc))) | |
| def _check_tasks(url: str, tally: Callable[[bool], None]) -> None: | |
| print("\n[2/7] Task Enumeration") | |
| try: | |
| response = requests.get(f"{url}/tasks", timeout=10) | |
| tally(check("GET /tasks returns 200", response.status_code == 200)) | |
| tasks = response.json().get("tasks", []) | |
| tally(check("3 tasks returned", len(tasks) == 3, f"got {len(tasks)}")) | |
| for task in tasks: | |
| tally(check(f"Task '{task['task_id']}' has action_schema", "action_schema" in task)) | |
| except requests.RequestException as exc: | |
| tally(check("Tasks endpoint works", False, str(exc))) | |
| def _check_loop(url: str, tally: Callable[[bool], None]) -> None: | |
| print("\n[3/7] Environment Loop (reset -> step -> state)") | |
| for task_id in TASK_IDS: | |
| print(f"\n --- {task_id} ---") | |
| try: | |
| reset_response = requests.post(f"{url}/reset", json={"task_id": task_id}, headers=API_HEADERS, timeout=10) | |
| tally(check(f"reset({task_id}) returns 200", reset_response.status_code == 200)) | |
| obs = reset_response.json() | |
| tally(check("Observation has timestep", "timestep" in obs)) | |
| tally(check("Observation has vitality_score", "vitality_score" in obs)) | |
| tally(check("Observation has file_tree", "file_tree" in obs and len(obs["file_tree"]) >= 8)) | |
| tally(check("Observation has test_results", "test_results" in obs and len(obs["test_results"]) >= 10)) | |
| tally(check("Observation has watchdog_flags", "watchdog_flags" in obs)) | |
| tally(check("Observation has active_checkpoints", "active_checkpoints" in obs)) | |
| state_response = requests.get(f"{url}/state", headers=API_HEADERS, timeout=10) | |
| tally(check("state() returns 200", state_response.status_code == 200)) | |
| state = state_response.json() | |
| tally(check("State shows not done", state.get("done") is False)) | |
| step_val = state.get("current_step") | |
| tally(check(f"State step == 0 (got {step_val})", step_val == 0)) | |
| tally(check("State has task_id", state.get("task_id") == task_id)) | |
| step_action = {"action_type": "emit_signal", "signal_type": "validation_ping"} | |
| step_response = requests.post(f"{url}/step", json=step_action, headers=API_HEADERS, timeout=10) | |
| tally(check("step() returns 200", step_response.status_code == 200)) | |
| result = step_response.json() | |
| tally(check("StepResult has reward", "reward" in result)) | |
| tally(check("Reward is float", isinstance(result["reward"], (int, float)))) | |
| tally(check("StepResult has done", "done" in result)) | |
| tally(check("StepResult has reward_breakdown", "reward_breakdown" in result)) | |
| reward_breakdown = result.get("reward_breakdown", {}) | |
| if reward_breakdown: | |
| tally(check("R1 vitality_delta present", "vitality_delta" in reward_breakdown)) | |
| tally(check("R2 test_recovery present", "test_recovery" in reward_breakdown)) | |
| tally(check("R3 efficiency_bonus present", "efficiency_bonus" in reward_breakdown)) | |
| tally(check("R4 coordination_bonus present", "coordination_bonus" in reward_breakdown)) | |
| tally(check("R5 novelty_bonus present", "novelty_bonus" in reward_breakdown)) | |
| tally(check("Watchdog penalty present", "watchdog_penalty" in reward_breakdown)) | |
| except requests.RequestException as exc: | |
| tally(check(f"{task_id} full loop", False, str(exc))) | |
| def _check_grader(url: str, tally: Callable[[bool], None]) -> None: | |
| print("\n[4/7] Grader") | |
| for task_id in TASK_IDS: | |
| try: | |
| actions = [ | |
| {"action_type": "emit_signal", "signal_type": "test"}, | |
| {"action_type": "emit_signal", "signal_type": "test"}, | |
| {"action_type": "do_nothing"}, | |
| ] | |
| response = requests.post( | |
| f"{url}/grader", | |
| json={"task_id": task_id, "actions": actions}, | |
| headers=API_HEADERS, | |
| timeout=10, | |
| ) | |
| tally(check(f"grader({task_id}) returns 200", response.status_code == 200)) | |
| result = response.json() | |
| score = result.get("score", -1) | |
| tally(check("Score in (0.0, 1.0)", 0.0 < score < 1.0, f"score={score}")) | |
| tally(check("survived field present", "survived" in result)) | |
| tally(check("watchdog_violations field present", "watchdog_violations" in result)) | |
| except requests.RequestException as exc: | |
| tally(check(f"Grader {task_id}", False, str(exc))) | |
| def _check_watchdog(url: str, tally: Callable[[bool], None]) -> None: | |
| print("\n[5/7] Watchdog Security") | |
| try: | |
| requests.post(f"{url}/reset", json={"task_id": "phase_1"}, headers=API_HEADERS, timeout=10) | |
| response = requests.post( | |
| f"{url}/step", | |
| json={"action_type": "patch_file", "path": "tests/test_core.py", "diff": "old|new"}, | |
| headers=API_HEADERS, | |
| timeout=10, | |
| ) | |
| result = response.json() | |
| reward_breakdown = result.get("reward_breakdown", {}) | |
| tally( | |
| check( | |
| "Watchdog penalizes protected file write", | |
| reward_breakdown.get("watchdog_penalty", 0) < 0, | |
| f"penalty={reward_breakdown.get('watchdog_penalty', 0)}", | |
| ) | |
| ) | |
| except requests.RequestException as exc: | |
| tally(check("Watchdog test", False, str(exc))) | |
| def _check_boundaries(url: str, tally: Callable[[bool], None]) -> None: | |
| print("\n[6/7] Episode Boundaries") | |
| try: | |
| requests.post(f"{url}/reset", json={"task_id": "phase_1"}, headers=API_HEADERS, timeout=10) | |
| for _ in range(25): | |
| response = requests.post( | |
| f"{url}/step", | |
| json={"action_type": "emit_signal", "signal_type": "test"}, | |
| headers=API_HEADERS, | |
| timeout=10, | |
| ) | |
| if response.json().get("done"): | |
| break | |
| after_done = requests.post( | |
| f"{url}/step", | |
| json={"action_type": "emit_signal", "signal_type": "test"}, | |
| headers=API_HEADERS, | |
| timeout=10, | |
| ) | |
| tally(check("Step after done returns done=True", after_done.json().get("done") is True)) | |
| except requests.RequestException as exc: | |
| tally(check("Episode boundary", False, str(exc))) | |
| def _check_schema(url: str, tally: Callable[[bool], None]) -> None: | |
| print("\n[7/7] Schema Endpoint") | |
| try: | |
| response = requests.get(f"{url}/schema", timeout=10) | |
| tally(check("GET /schema returns 200", response.status_code == 200)) | |
| schema = response.json() | |
| tally(check("Schema has action", "action" in schema)) | |
| tally(check("Schema has observation", "observation" in schema)) | |
| tally(check("Schema has state", "state" in schema)) | |
| except requests.RequestException as exc: | |
| tally(check("Schema endpoint", False, str(exc))) | |
| def main() -> None: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--api-url", default="http://localhost:7860") | |
| parser.add_argument("--api-key", default=os.environ.get("CODEORGANISM_API_KEY") or os.environ.get("CODEORGANISM_API_KEYS", "").split(",", 1)[0]) | |
| args = parser.parse_args() | |
| url = args.api_url.rstrip("/") | |
| if args.api_key: | |
| API_HEADERS["x-api-key"] = args.api_key | |
| passed = 0 | |
| failed = 0 | |
| total = 0 | |
| def tally(ok: bool) -> None: | |
| nonlocal passed, failed, total | |
| total += 1 | |
| if ok: | |
| passed += 1 | |
| else: | |
| failed += 1 | |
| sep = "=" * 60 | |
| print(f"\n{sep}") | |
| print(" CodeOrganismVM OpenEnv Pre-Submission Validator") | |
| print(f" Target: {url}") | |
| print(f"{sep}\n") | |
| _check_health(url, tally) | |
| _check_tasks(url, tally) | |
| _check_loop(url, tally) | |
| _check_grader(url, tally) | |
| _check_watchdog(url, tally) | |
| _check_boundaries(url, tally) | |
| _check_schema(url, tally) | |
| print(f"\n{sep}") | |
| print(f" Results: {passed}/{total} passed, {failed} failed") | |
| exit_code = min(failed, 1) | |
| status_messages = { | |
| 0: " \033[92mALL CHECKS PASSED — ready for submission!\033[0m", | |
| 1: f" \033[91m{failed} CHECKS FAILED — fix before submitting\033[0m", | |
| } | |
| print(status_messages[exit_code]) | |
| print(f"{sep}\n") | |
| sys.exit(exit_code) | |
| main() | |