Fix HF Space config and validator flow

Dockerfile

@@ -8,5 +8,8 @@ RUN pip install --no-cache-dir -r requirements.txt
 COPY . .
 
 ENV OFFLINE_INFERENCE=1
+ENV PORT=7860
 
-CMD ["python", "inference.py", "--max-steps", "8", "--policy-mode", "imp", "--trajectories", "4", "--force-local-env"]
+EXPOSE 7860
+
+CMD ["python", "-m", "uvicorn", "server.app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,3 +1,12 @@
+---
+title: CICD_DEBUGGER
+colorFrom: blue
+colorTo: green
+sdk: docker
+app_port: 7860
+pinned: false
+---
+
 # CI/CD Pipeline Debugger Environment (OpenEnv)
 
 ## 1. Project Goal
@@ -196,7 +205,7 @@ Coverage includes:
 OpenEnv validation:
 
 ```bash
-openenv validate
+python -m openenv.cli.__main__ validate
 ```
 
 Pre-submission script:
@@ -205,13 +214,28 @@ Pre-submission script:
 ./validate-submission.sh <your_hf_space_url>
 ```
 
-Docker run:
+Required environment variables:
+
+```bash
+export API_BASE_URL="https://router.huggingface.co/v1"
+export MODEL_NAME="Qwen/Qwen2.5-72B-Instruct"
+export HF_TOKEN="<your_token>"
+```
+
+Docker run (Space/API mode):
 
 ```bash
 docker build -t cicd-debugger-env .
-docker run --rm -e OFFLINE_INFERENCE=1 cicd-debugger-env
+docker run --rm -p 7860:7860 cicd-debugger-env
 ```
 
+Server endpoints used by validators:
+
+- `POST /reset`
+- `POST /step`
+- `GET /state`
+- `GET /health`
+
 ## 11. One-line Presentation Summary
 
 We built an OpenEnv-compliant reinforcement learning environment where AI agents learn to debug real CI/CD pipelines using multi-step reasoning, hybrid grading, anti-hacking safeguards, and robust reward shaping.

env/rewards.py

@@ -70,7 +70,7 @@ class RewardCalculator:
         )
         reward += self._penalty_reward(state=state, result=result, current_config=current_config)
 
-        return round(float(reward), 4)
+        return round(self._clamp_01(reward), 4)
 
     def _progress_reward(self, action: str, result: dict[str, Any]) -> float:
         reward = self.ACTION_PROGRESS_REWARDS.get(action, 0.0)

inference.py

@@ -529,7 +529,7 @@ async def run_episode(args: argparse.Namespace) -> int:
                 metadata={"broken_token": "npm tset", "fixed_token": "npm test"},
             )
 
-            combined_reward = round(float(calculated_reward) + float(env_reward), 4)
+            combined_reward = round(max(0.0, min(1.0, float(calculated_reward) + float(env_reward))), 4)
             metrics.add_step(action=action_text, reward=combined_reward, error=step_error, done=done)
             steps_taken = step
 

server/app.py

@@ -1,19 +1,258 @@
 from __future__ import annotations
 
+from dataclasses import dataclass
+import os
+
 from fastapi import FastAPI
+from fastapi import HTTPException
+from pydantic import BaseModel, Field
 import uvicorn
 
 
 app = FastAPI(title="CI/CD Debugger OpenEnv Server")
 
 
+DEFAULT_ORIGINAL_CONFIG = """
+name: CI
+on: [push]
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: npm ci
+      - run: npm tset
+""".strip()
+
+DEFAULT_EXPECTED_CONFIG = """
+name: CI
+on: [push]
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: npm ci
+      - run: npm test
+""".strip()
+
+DEFAULT_ERROR_MESSAGE = "command not found"
+
+
+class ObservationModel(BaseModel):
+    config: str
+    error_message: str
+    logs: str
+    last_action_error: str | None = None
+
+
+class ResetRequest(BaseModel):
+    task_id: str = "cicd-debugger-task"
+    original_config: str | None = None
+    expected_config: str | None = None
+    error_message: str | None = None
+
+
+class StepRequest(BaseModel):
+    action: str = Field(min_length=1, max_length=600)
+
+
+class StepResponse(BaseModel):
+    task_id: str
+    step_count: int
+    reward: float
+    done: bool
+    observation: ObservationModel
+    last_action: str | None = None
+
+
+class StateResponse(BaseModel):
+    initialized: bool
+    task_id: str | None = None
+    step_count: int = 0
+    done: bool = False
+    last_action: str | None = None
+    observation: ObservationModel | None = None
+
+
+@dataclass
+class LocalObservation:
+    config: str
+    error_message: str
+    logs: str
+    last_action_error: str | None = None
+
+
+@dataclass
+class LocalStepResult:
+    observation: LocalObservation
+    reward: float
+    done: bool
+    last_action_error: str | None = None
+
+
+class LocalCICDDebuggerEnv:
+    def __init__(self, original_config: str, expected_config: str, error_message: str):
+        self.original_config = original_config
+        self.expected_config = expected_config
+        self.error_message = error_message
+        self.current_config = original_config
+
+    def reset(self) -> LocalStepResult:
+        self.current_config = self.original_config
+        observation = LocalObservation(
+            config=self.current_config,
+            error_message=self.error_message,
+            logs="CI failed in test step: npm tset is not a valid command.",
+        )
+        return LocalStepResult(observation=observation, reward=0.0, done=False)
+
+    def step(self, action: str) -> LocalStepResult:
+        action_text = str(action or "").strip()
+        lower_action = action_text.lower()
+        previous = self.current_config
+        step_error: str | None = None
+        logs = "No effective change applied."
+
+        if _is_hacking_action(action_text):
+            step_error = "disallowed_hacking_pattern"
+            logs = "Rejected unsafe action pattern."
+        elif "npm tset" in lower_action and "npm test" in lower_action and "npm tset" in previous:
+            self.current_config = previous.replace("npm tset", "npm test")
+            logs = "Patched CI command typo from npm tset to npm test."
+        elif "replace" in lower_action and "npm test" in lower_action and "npm tset" in previous:
+            self.current_config = previous.replace("npm tset", "npm test")
+            logs = "Applied replace operation for broken test command."
+        elif "npm test" in lower_action and "npm tset" in previous:
+            self.current_config = previous.replace("npm tset", "npm test")
+            logs = "Applied inferred command fix."
+
+        done = "npm tset" not in self.current_config.lower() and "npm test" in self.current_config.lower()
+        reward = 1.0 if done else 0.0
+        error_message = "" if done else self.error_message
+
+        observation = LocalObservation(
+            config=self.current_config,
+            error_message=error_message,
+            logs=logs,
+            last_action_error=step_error,
+        )
+        return LocalStepResult(observation=observation, reward=reward, done=done, last_action_error=step_error)
+
+
+@dataclass
+class RuntimeSession:
+    task_id: str
+    env: LocalCICDDebuggerEnv
+    step_count: int = 0
+    done: bool = False
+    last_action: str | None = None
+    last_result: LocalStepResult | None = None
+
+
+runtime_session: RuntimeSession | None = None
+
+
+def _is_hacking_action(action_text: str) -> bool:
+    value = (action_text or "").lower()
+    patterns = (
+        "if: false",
+        "when: never",
+        "echo \"tests passed\"",
+        "echo 'tests passed'",
+        "exit 0",
+        "force success",
+    )
+    return any(token in value for token in patterns)
+
+
+def _as_observation_model(observation: LocalObservation) -> ObservationModel:
+    return ObservationModel(
+        config=observation.config,
+        error_message=observation.error_message,
+        logs=observation.logs,
+        last_action_error=observation.last_action_error,
+    )
+
+
+def _build_step_response(session: RuntimeSession, result: LocalStepResult) -> StepResponse:
+    return StepResponse(
+        task_id=session.task_id,
+        step_count=session.step_count,
+        reward=float(result.reward),
+        done=bool(result.done),
+        observation=_as_observation_model(result.observation),
+        last_action=session.last_action,
+    )
+
+
 @app.get("/health")
 def health() -> dict[str, str]:
     return {"status": "ok"}
 
 
+@app.post("/reset", response_model=StepResponse)
+def reset(payload: ResetRequest | None = None) -> StepResponse:
+    global runtime_session
+
+    request = payload or ResetRequest()
+    env = LocalCICDDebuggerEnv(
+        original_config=request.original_config or DEFAULT_ORIGINAL_CONFIG,
+        expected_config=request.expected_config or DEFAULT_EXPECTED_CONFIG,
+        error_message=request.error_message or DEFAULT_ERROR_MESSAGE,
+    )
+    result = env.reset()
+
+    runtime_session = RuntimeSession(task_id=request.task_id, env=env, last_result=result)
+    return _build_step_response(runtime_session, result)
+
+
+@app.post("/step", response_model=StepResponse)
+def step(payload: StepRequest) -> StepResponse:
+    global runtime_session
+
+    if runtime_session is None:
+        raise HTTPException(status_code=400, detail="Environment not initialized. Call /reset first.")
+
+    if runtime_session.done and runtime_session.last_result is not None:
+        return _build_step_response(runtime_session, runtime_session.last_result)
+
+    result = runtime_session.env.step(payload.action)
+    runtime_session.step_count += 1
+    runtime_session.done = bool(result.done)
+    runtime_session.last_action = payload.action
+    runtime_session.last_result = result
+
+    return _build_step_response(runtime_session, result)
+
+
+@app.get("/state", response_model=StateResponse)
+def state() -> StateResponse:
+    if runtime_session is None:
+        return StateResponse(initialized=False)
+
+    observation = None
+    if runtime_session.last_result is not None:
+        observation = _as_observation_model(runtime_session.last_result.observation)
+
+    return StateResponse(
+        initialized=True,
+        task_id=runtime_session.task_id,
+        step_count=runtime_session.step_count,
+        done=runtime_session.done,
+        last_action=runtime_session.last_action,
+        observation=observation,
+    )
+
+
+@app.post("/state", response_model=StateResponse)
+def state_post() -> StateResponse:
+    return state()
+
+
 def main() -> None:
-    uvicorn.run(app, host="0.0.0.0", port=8000)
+    port = int(os.getenv("PORT", "7860"))
+    uvicorn.run(app, host="0.0.0.0", port=port)
 
 
 if __name__ == "__main__":

tests/test_day2_engine.py

@@ -191,7 +191,8 @@ jobs:
             expected_config=EXPECTED_CONFIG,
         )
 
-        self.assertLess(reward, 0.0)
+        self.assertGreaterEqual(reward, 0.0)
+        self.assertLessEqual(reward, 0.3)
 
 
 if __name__ == "__main__":

tests/test_server_api.py

@@ -0,0 +1,49 @@
+import unittest
+
+from fastapi.testclient import TestClient
+
+from server.app import app
+import server.app as server_app
+
+
+class ServerApiTests(unittest.TestCase):
+    def setUp(self):
+        server_app.runtime_session = None
+        self.client = TestClient(app)
+
+    def test_health(self):
+        response = self.client.get("/health")
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json().get("status"), "ok")
+
+    def test_reset_state_step_flow(self):
+        reset_response = self.client.post("/reset", json={})
+        self.assertEqual(reset_response.status_code, 200)
+        reset_payload = reset_response.json()
+        self.assertIn("observation", reset_payload)
+        self.assertIn("step_count", reset_payload)
+        self.assertEqual(reset_payload["step_count"], 0)
+
+        state_response = self.client.get("/state")
+        self.assertEqual(state_response.status_code, 200)
+        state_payload = state_response.json()
+        self.assertTrue(state_payload.get("initialized"))
+
+        step_response = self.client.post(
+            "/step",
+            json={"action": "edit_config: replace npm tset with npm test"},
+        )
+        self.assertEqual(step_response.status_code, 200)
+        step_payload = step_response.json()
+        self.assertIn("reward", step_payload)
+        self.assertIn("done", step_payload)
+
+    def test_step_requires_reset(self):
+        server_app.runtime_session = None
+        client = TestClient(app)
+        response = client.post("/step", json={"action": "read_logs: inspect logs"})
+        self.assertEqual(response.status_code, 400)
+
+
+if __name__ == "__main__":
+    unittest.main()

validate-submission.sh

@@ -2,7 +2,7 @@
 #
 # validate-submission.sh - OpenEnv Submission Validator
 #
-# Checks that your HF Space is live, Docker image builds, and openenv validate passes.
+# Checks all mandatory pre-submission gates for this challenge.
 #
 # Usage:
 #   ./validate-submission.sh <ping_url> [repo_dir]
@@ -10,6 +10,7 @@
 set -uo pipefail
 
 DOCKER_BUILD_TIMEOUT=600
+INFERENCE_TIMEOUT=1200
 if [ -t 1 ]; then
   RED='\033[0;31m'
   GREEN='\033[0;32m'
@@ -66,6 +67,19 @@ fi
 
 PING_URL="${PING_URL%/}"
 PASS=0
+PYTHON_CMD="python3"
+if [ -x "$REPO_DIR/.venv/bin/python" ]; then
+  PYTHON_CMD="$REPO_DIR/.venv/bin/python"
+fi
+
+run_openenv_validate() {
+  if command -v openenv &>/dev/null; then
+    if (cd "$REPO_DIR" && openenv validate); then
+      return 0
+    fi
+  fi
+  (cd "$REPO_DIR" && "$PYTHON_CMD" -m openenv.cli.__main__ validate)
+}
 
 log()  { printf "[%s] %b\n" "$(date -u +%H:%M:%S)" "$*"; }
 pass() { log "${GREEN}PASSED${NC} -- $1"; PASS=$((PASS + 1)); }
@@ -83,9 +97,43 @@ printf "${BOLD}  OpenEnv Submission Validator${NC}\n"
 printf "${BOLD}========================================${NC}\n"
 log "Repo:     $REPO_DIR"
 log "Ping URL: $PING_URL"
+log "Python:   $PYTHON_CMD"
 printf "\n"
 
-log "${BOLD}Step 1/3: Pinging HF Space${NC} ($PING_URL/reset) ..."
+log "${BOLD}Step 1/8: Checking mandatory environment variables${NC} ..."
+
+REQUIRED_VARS=(API_BASE_URL MODEL_NAME HF_TOKEN)
+MISSING_VARS=()
+for var_name in "${REQUIRED_VARS[@]}"; do
+  if [ -z "${!var_name:-}" ]; then
+    MISSING_VARS+=("$var_name")
+  fi
+done
+
+if [ ${#MISSING_VARS[@]} -eq 0 ]; then
+  pass "Required environment variables are set (API_BASE_URL, MODEL_NAME, HF_TOKEN)"
+else
+  fail "Missing required environment variables: ${MISSING_VARS[*]}"
+  hint "Export them before submission, for example: export API_BASE_URL=... MODEL_NAME=... HF_TOKEN=..."
+  stop_at "Step 1"
+fi
+
+log "${BOLD}Step 2/8: Verifying inference entrypoint and OpenAI client usage${NC} ..."
+
+if [ ! -f "$REPO_DIR/inference.py" ]; then
+  fail "inference.py not found at repository root"
+  stop_at "Step 2"
+fi
+
+if grep -Eq "from[[:space:]]+openai[[:space:]]+import[[:space:]]+OpenAI|openai\\.OpenAI|OpenAI\\(" "$REPO_DIR/inference.py"; then
+  pass "Root inference.py exists and uses OpenAI client"
+else
+  fail "Could not verify OpenAI client usage in inference.py"
+  hint "Use: from openai import OpenAI and initialize client with API_BASE_URL/MODEL_NAME/HF_TOKEN"
+  stop_at "Step 2"
+fi
+
+log "${BOLD}Step 3/8: Pinging HF Space endpoints${NC} ($PING_URL/reset, /state, /step) ..."
 
 CURL_OUTPUT=$(portable_mktemp "validate-curl")
 CLEANUP_FILES+=("$CURL_OUTPUT")
@@ -99,19 +147,49 @@ elif [ "$HTTP_CODE" = "000" ]; then
   fail "HF Space not reachable (connection failed or timed out)"
   hint "Check your network connection and that the Space is running."
   hint "Try: curl -s -o /dev/null -w '%{http_code}' -X POST $PING_URL/reset"
-  stop_at "Step 1"
+  stop_at "Step 3"
 else
   fail "HF Space /reset returned HTTP $HTTP_CODE (expected 200)"
   hint "Make sure your Space is running and the URL is correct."
-  stop_at "Step 1"
+  stop_at "Step 3"
+fi
+
+STATE_OUTPUT=$(portable_mktemp "validate-state")
+CLEANUP_FILES+=("$STATE_OUTPUT")
+STATE_CODE=$(curl -s -o "$STATE_OUTPUT" -w "%{http_code}" "$PING_URL/state" --max-time 30 || printf "000")
+if [ "$STATE_CODE" != "200" ]; then
+  STATE_CODE=$(curl -s -o "$STATE_OUTPUT" -w "%{http_code}" -X POST \
+    -H "Content-Type: application/json" -d '{}' \
+    "$PING_URL/state" --max-time 30 || printf "000")
+fi
+
+if [ "$STATE_CODE" = "200" ]; then
+  pass "HF Space responds to /state"
+else
+  fail "HF Space /state returned HTTP $STATE_CODE (expected 200)"
+  stop_at "Step 3"
+fi
+
+STEP_OUTPUT=$(portable_mktemp "validate-step")
+CLEANUP_FILES+=("$STEP_OUTPUT")
+STEP_CODE=$(curl -s -o "$STEP_OUTPUT" -w "%{http_code}" -X POST \
+  -H "Content-Type: application/json" \
+  -d '{"action":"read_logs: inspect failing stage logs"}' \
+  "$PING_URL/step" --max-time 30 || printf "000")
+
+if [ "$STEP_CODE" = "200" ]; then
+  pass "HF Space responds to /step"
+else
+  fail "HF Space /step returned HTTP $STEP_CODE (expected 200)"
+  stop_at "Step 3"
 fi
 
-log "${BOLD}Step 2/3: Running docker build${NC} ..."
+log "${BOLD}Step 4/8: Running docker build${NC} ..."
 
 if ! command -v docker &>/dev/null; then
   fail "docker command not found"
   hint "Install Docker: https://docs.docker.com/get-docker/"
-  stop_at "Step 2"
+  stop_at "Step 4"
 fi
 
 if [ -f "$REPO_DIR/Dockerfile" ]; then
@@ -120,7 +198,7 @@ elif [ -f "$REPO_DIR/server/Dockerfile" ]; then
   DOCKER_CONTEXT="$REPO_DIR/server"
 else
   fail "No Dockerfile found in repo root or server/ directory"
-  stop_at "Step 2"
+  stop_at "Step 4"
 fi
 
 BUILD_OK=false
@@ -131,31 +209,192 @@ if [ "$BUILD_OK" = true ]; then
 else
   fail "Docker build failed (timeout=${DOCKER_BUILD_TIMEOUT}s)"
   printf "%s\n" "$BUILD_OUTPUT" | tail -20
-  stop_at "Step 2"
+  stop_at "Step 4"
 fi
 
-log "${BOLD}Step 3/3: Running openenv validate${NC} ..."
-
-if ! command -v openenv &>/dev/null; then
-  fail "openenv command not found"
-  hint "Install it: pip install openenv-core"
-  stop_at "Step 3"
-fi
+log "${BOLD}Step 5/8: Running openenv validate${NC} ..."
 
 VALIDATE_OK=false
-VALIDATE_OUTPUT=$(cd "$REPO_DIR" && openenv validate 2>&1) && VALIDATE_OK=true
+VALIDATE_OUTPUT=$(run_openenv_validate 2>&1) && VALIDATE_OK=true
 
 if [ "$VALIDATE_OK" = true ]; then
   pass "openenv validate passed"
 else
   fail "openenv validate failed"
   printf "%s\n" "$VALIDATE_OUTPUT"
-  stop_at "Step 3"
+  stop_at "Step 5"
+fi
+
+log "${BOLD}Step 6/8: Running baseline inference under 20-minute limit${NC} ..."
+
+INFER_OK=false
+INFER_OUTPUT=$(run_with_timeout "$INFERENCE_TIMEOUT" env OFFLINE_INFERENCE=1 \
+  "$PYTHON_CMD" "$REPO_DIR/inference.py" --max-steps 8 --offline --force-local-env 2>&1) && INFER_OK=true
+
+if [ "$INFER_OK" != true ]; then
+  fail "inference.py failed or exceeded ${INFERENCE_TIMEOUT}s"
+  printf "%s\n" "$INFER_OUTPUT" | tail -40
+  stop_at "Step 6"
+fi
+
+NON_EMPTY_OUTPUT=$(printf "%s\n" "$INFER_OUTPUT" | sed '/^[[:space:]]*$/d')
+FIRST_LINE=$(printf "%s\n" "$NON_EMPTY_OUTPUT" | head -n 1)
+LAST_LINE=$(printf "%s\n" "$NON_EMPTY_OUTPUT" | tail -n 1)
+
+if ! printf "%s\n" "$FIRST_LINE" | grep -Eq '^\[START\] task=[^ ]+ env=[^ ]+ model=.+$'; then
+  fail "Missing or malformed [START] line"
+  printf "%s\n" "$INFER_OUTPUT" | tail -40
+  stop_at "Step 6"
+fi
+
+if ! printf "%s\n" "$NON_EMPTY_OUTPUT" | grep -Eq '^\[STEP\] step=[0-9]+ action=.* reward=-?[0-9]+\.[0-9]{2} done=(true|false) error=(null|.+)$'; then
+  fail "Missing or malformed [STEP] line"
+  printf "%s\n" "$INFER_OUTPUT" | tail -40
+  stop_at "Step 6"
+fi
+
+if ! printf "%s\n" "$LAST_LINE" | grep -Eq '^\[END\] success=(true|false) steps=[0-9]+ rewards=(-?[0-9]+\.[0-9]{2}(,-?[0-9]+\.[0-9]{2})*)?$'; then
+  fail "Missing or malformed [END] line"
+  printf "%s\n" "$INFER_OUTPUT" | tail -40
+  stop_at "Step 6"
+fi
+
+pass "Baseline inference completed and emitted strict [START]/[STEP]/[END] format"
+
+log "${BOLD}Step 7/8: Validating 3+ tasks with graders (scores/reward in 0.0-1.0)${NC} ..."
+
+PY_CHECK_SCRIPT=$(portable_mktemp "validate-grader")
+CLEANUP_FILES+=("$PY_CHECK_SCRIPT")
+cat >"$PY_CHECK_SCRIPT" <<'PY'
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import yaml
+
+repo = Path(sys.argv[1]).resolve()
+sys.path.insert(0, str(repo))
+
+from env.graders.deterministic import DeterministicGrader
+from env.hidden_tests import HiddenTestRunner
+from env.rewards import RewardCalculator
+
+
+def build_config(command: str) -> str:
+    return f"""
+name: CI
+on: [push]
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: npm ci
+      - run: {command}
+""".strip()
+
+
+config_path = repo / "openenv.yaml"
+data = yaml.safe_load(config_path.read_text(encoding="utf-8"))
+tasks = data.get("tasks", [])
+if len(tasks) < 3:
+    raise SystemExit("expected at least 3 tasks in openenv.yaml")
+
+grader = DeterministicGrader()
+hidden = HiddenTestRunner(grader=grader)
+reward_calculator = RewardCalculator()
+
+results = []
+for task in tasks[:3]:
+    metadata = dict(task.get("metadata") or {})
+    broken_token = str(metadata.get("broken_token", "npm tset"))
+    fixed_token = str(metadata.get("fixed_token", "npm test"))
+
+    original_config = build_config(broken_token)
+    fixed_config = build_config(fixed_token)
+
+    deterministic_score = float(grader.grade(fixed_config, fixed_config, metadata))
+    hidden_score = float(hidden.evaluate_fix(fixed_config, expected_config=fixed_config, metadata=metadata))
+    reward_score = float(
+        reward_calculator.calculate_step_reward(
+            state={
+                "step_count": 1,
+                "previous_config": original_config,
+                "expected_config": fixed_config,
+                "original_config": original_config,
+                "error": "command not found",
+            },
+            action="validate_fix",
+            result={
+                "previous_config": original_config,
+                "current_config": fixed_config,
+                "expected_config": fixed_config,
+                "logs_analyzed": True,
+                "error_diagnosed": True,
+                "fix_proposed": True,
+                "pipeline_run": True,
+                "tests_passed": True,
+                "command_succeeded": True,
+                "changed_files_count": 1,
+                "changed_lines_count": 2,
+            },
+            original_config=original_config,
+            fixed_config=fixed_config,
+            error_message="command not found",
+            expected_config=fixed_config,
+            metadata=metadata,
+        )
+    )
+
+    for label, value in (
+        ("deterministic", deterministic_score),
+        ("hidden", hidden_score),
+        ("reward", reward_score),
+    ):
+        if value < 0.0 or value > 1.0:
+            raise SystemExit(f"{task.get('id', 'unknown')} {label} out of range: {value}")
+
+    results.append(
+        {
+            "id": task.get("id", "unknown"),
+            "deterministic": round(deterministic_score, 4),
+            "hidden": round(hidden_score, 4),
+            "reward": round(reward_score, 4),
+        }
+    )
+
+print(json.dumps(results, indent=2, sort_keys=True))
+PY
+
+GRADER_OK=false
+GRADER_OUTPUT=$(run_with_timeout 180 "$PYTHON_CMD" "$PY_CHECK_SCRIPT" "$REPO_DIR" 2>&1) && GRADER_OK=true
+
+if [ "$GRADER_OK" = true ]; then
+  pass "Grader checks passed on 3 tasks with scores/reward in [0.0, 1.0]"
+else
+  fail "Grader range check failed"
+  printf "%s\n" "$GRADER_OUTPUT"
+  stop_at "Step 7"
+fi
+
+log "${BOLD}Step 8/8: Verifying unit test suite${NC} ..."
+
+TESTS_OK=false
+TESTS_OUTPUT=$(run_with_timeout 300 "$PYTHON_CMD" -m unittest discover -s "$REPO_DIR/tests" -v 2>&1) && TESTS_OK=true
+
+if [ "$TESTS_OK" = true ]; then
+  pass "Unit tests passed"
+else
+  fail "Unit tests failed"
+  printf "%s\n" "$TESTS_OUTPUT" | tail -40
+  stop_at "Step 8"
 fi
 
 printf "\n"
 printf "${BOLD}========================================${NC}\n"
-printf "${GREEN}${BOLD}  All 3/3 checks passed!${NC}\n"
+printf "${GREEN}${BOLD}  All 8/8 checks passed!${NC}\n"
 printf "${GREEN}${BOLD}  Your submission is ready to submit.${NC}\n"
 printf "${BOLD}========================================${NC}\n"
 printf "\n"