test scripts and change in inference.py

inference.original.py

@@ -0,0 +1,197 @@
+"""
+Inference Script Example
+===================================
+MANDATORY
+- Before submitting, ensure the following variables are defined in your environment configuration:
+    API_BASE_URL   The API endpoint for the LLM.
+    MODEL_NAME     The model identifier to use for inference.
+    HF_TOKEN       Your Hugging Face / API key.
+    LOCAL_IMAGE_NAME The name of the local image to use for the environment if you are using from_docker_image()
+                     method
+
+- Defaults are set only for API_BASE_URL and MODEL_NAME
+    (and should reflect your active inference setup):
+    API_BASE_URL = os.getenv("API_BASE_URL", "<your-active-endpoint>")
+    MODEL_NAME = os.getenv("MODEL_NAME", "<your-active-model>")
+
+- The inference script must be named `inference.py` and placed in the root directory of the project
+- Participants must use OpenAI Client for all LLM calls using above variables
+
+STDOUT FORMAT
+- The script must emit exactly three line types to stdout, in this order:
+
+    [START] task=<task_name> env=<benchmark> model=<model_name>
+    [STEP]  step=<n> action=<action_str> reward=<0.00> done=<true|false> error=<msg|null>
+    [END]   success=<true|false> steps=<n> score=<score> rewards=<r1,r2,...,rn>
+
+  Rules:
+    - One [START] line at episode begin.
+    - One [STEP] line per step, immediately after env.step() returns.
+    - One [END] line after env.close(), always emitted (even on exception).
+    - reward and rewards are formatted to 2 decimal places.
+    - done and success are lowercase booleans: true or false.
+    - error is the raw last_action_error string, or null if none.
+    - All fields on a single line with no newlines within a line.
+    - Each tasks should return score in [0, 1]
+
+  Example:
+    [START] task=click-test env=miniwob model=Qwen3-VL-30B
+    [STEP] step=1 action=click('123') reward=0.00 done=false error=null
+    [STEP] step=2 action=fill('456','text') reward=0.00 done=false error=null
+    [STEP] step=3 action=click('789') reward=1.00 done=true error=null
+    [END] success=true steps=3 score=1.00 rewards=0.00,0.00,1.00
+"""
+
+import asyncio
+import os
+import textwrap
+from typing import List, Optional
+
+from my_env_v4 import MyEnvV4Action, MyEnvV4Env
+from openai import OpenAI
+
+IMAGE_NAME = os.getenv("IMAGE_NAME")  # If you are using docker image
+API_KEY = os.getenv("HF_TOKEN") or os.getenv("API_KEY")
+
+API_BASE_URL = os.getenv("API_BASE_URL") or "https://router.huggingface.co/v1"
+MODEL_NAME = os.getenv("MODEL_NAME") or "Qwen/Qwen2.5-72B-Instruct"
+TASK_NAME = os.getenv("MY_ENV_V4_TASK", "echo")
+BENCHMARK = os.getenv("MY_ENV_V4_BENCHMARK", "my_env_v4")
+MAX_STEPS = 8
+TEMPERATURE = 0.7
+MAX_TOKENS = 150
+SUCCESS_SCORE_THRESHOLD = 0.1  # normalized score in [0, 1]
+
+# Max possible reward: each token contributes 0.1, across all steps
+_MAX_REWARD_PER_STEP = MAX_TOKENS * 0.1
+MAX_TOTAL_REWARD = MAX_STEPS * _MAX_REWARD_PER_STEP
+
+SYSTEM_PROMPT = textwrap.dedent(
+    """
+    You are interacting with a simple echo environment.
+    Each turn you must send a message. The environment will echo it back.
+    Reward is proportional to message length: reward = len(message) * 0.1
+    Your goal is to maximize total reward by sending meaningful, substantive messages.
+    Reply with exactly one message string — no quotes, no prefixes, just the message text.
+    """
+).strip()
+
+
+def log_start(task: str, env: str, model: str) -> None:
+    print(f"[START] task={task} env={env} model={model}", flush=True)
+
+
+def log_step(
+    step: int, action: str, reward: float, done: bool, error: Optional[str]
+) -> None:
+    error_val = error if error else "null"
+    done_val = str(done).lower()
+    print(
+        f"[STEP] step={step} action={action} reward={reward:.2f} done={done_val} error={error_val}",
+        flush=True,
+    )
+
+
+def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
+    rewards_str = ",".join(f"{r:.2f}" for r in rewards)
+    print(
+        f"[END] success={str(success).lower()} steps={steps} score={score:.3f} rewards={rewards_str}",
+        flush=True,
+    )
+
+
+def build_user_prompt(
+    step: int, last_echoed: str, last_reward: float, history: List[str]
+) -> str:
+    history_block = "\n".join(history[-4:]) if history else "None"
+    return textwrap.dedent(
+        f"""
+        Step: {step}
+        Last echoed message: {last_echoed!r}
+        Last reward: {last_reward:.2f}
+        Previous steps:
+        {history_block}
+        Send your next message.
+        """
+    ).strip()
+
+
+def get_model_message(
+    client: OpenAI, step: int, last_echoed: str, last_reward: float, history: List[str]
+) -> str:
+    user_prompt = build_user_prompt(step, last_echoed, last_reward, history)
+    try:
+        completion = client.chat.completions.create(
+            model=MODEL_NAME,
+            messages=[
+                {"role": "system", "content": SYSTEM_PROMPT},
+                {"role": "user", "content": user_prompt},
+            ],
+            temperature=TEMPERATURE,
+            max_tokens=MAX_TOKENS,
+            stream=False,
+        )
+        text = (completion.choices[0].message.content or "").strip()
+        return text if text else "hello"
+    except Exception as exc:
+        print(f"[DEBUG] Model request failed: {exc}", flush=True)
+        return "hello"
+
+
+async def main() -> None:
+    client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
+
+    env = await MyEnvV4Env.from_docker_image(IMAGE_NAME)
+
+    history: List[str] = []
+    rewards: List[float] = []
+    steps_taken = 0
+    score = 0.0
+    success = False
+
+    log_start(task=TASK_NAME, env=BENCHMARK, model=MODEL_NAME)
+
+    try:
+        result = await env.reset()  # OpenENV.reset()
+        last_echoed = result.observation.echoed_message
+        last_reward = 0.0
+
+        for step in range(1, MAX_STEPS + 1):
+            if result.done:
+                break
+
+            message = get_model_message(client, step, last_echoed, last_reward, history)
+
+            result = await env.step(MyEnvV4Action(message=message))
+            obs = result.observation
+
+            reward = result.reward or 0.0
+            done = result.done
+            error = None
+
+            rewards.append(reward)
+            steps_taken = step
+            last_echoed = obs.echoed_message
+            last_reward = reward
+
+            log_step(step=step, action=message, reward=reward, done=done, error=error)
+
+            history.append(f"Step {step}: {message!r} -> reward {reward:+.2f}")
+
+            if done:
+                break
+
+        score = sum(rewards) / MAX_TOTAL_REWARD if MAX_TOTAL_REWARD > 0 else 0.0
+        score = min(max(score, 0.0), 1.0)  # clamp to [0, 1]
+        success = score >= SUCCESS_SCORE_THRESHOLD
+
+    finally:
+        try:
+            await env.close()
+        except Exception as e:
+            print(f"[DEBUG] env.close() error (container cleanup): {e}", flush=True)
+        log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

inference.py

@@ -1,18 +1,47 @@
 """
-Inference Script for Curator Environment
-============================================
+Inference Script for Curator Environment (Docker version)
+=========================================================
+Loads the environment from a Docker image via from_docker_image().
 
-Environment Variables:
+MANDATORY
+- Before submitting, ensure the following variables are defined in your environment configuration:
     API_BASE_URL   The API endpoint for the LLM.
     MODEL_NAME     The model identifier to use for inference.
     HF_TOKEN       Your Hugging Face / API key.
-    IMAGE_NAME     Docker image name for the environment.
-    CURATOR_TASK  Task difficulty: "easy", "medium", or "hard" (default: "easy").
+    LOCAL_IMAGE_NAME The name of the local image to use for the environment if you are using from_docker_image()
+                     method
 
-STDOUT FORMAT:
-    [START] task=<task_name> env=curator model=<model_name>
+- Defaults are set only for API_BASE_URL and MODEL_NAME
+    (and should reflect your active inference setup):
+    API_BASE_URL = os.getenv("API_BASE_URL", "<your-active-endpoint>")
+    MODEL_NAME = os.getenv("MODEL_NAME", "<your-active-model>")
+
+- The inference script must be named `inference.py` and placed in the root directory of the project
+- Participants must use OpenAI Client for all LLM calls using above variables
+
+STDOUT FORMAT
+- The script must emit exactly three line types to stdout, in this order:
+
+    [START] task=<task_name> env=<benchmark> model=<model_name>
     [STEP]  step=<n> action=<action_str> reward=<0.00> done=<true|false> error=<msg|null>
-    [END]   success=<true|false> steps=<n> score=<0.000> rewards=<r1,r2,...,rn>
+    [END]   success=<true|false> steps=<n> score=<score> rewards=<r1,r2,...,rn>
+
+  Rules:
+    - One [START] line at episode begin.
+    - One [STEP] line per step, immediately after env.step() returns.
+    - One [END] line after env.close(), always emitted (even on exception).
+    - reward and rewards are formatted to 2 decimal places.
+    - done and success are lowercase booleans: true or false.
+    - error is the raw last_action_error string, or null if none.
+    - All fields on a single line with no newlines within a line.
+    - Each tasks should return score in [0, 1]
+
+  Example:
+    [START] task=click-test env=miniwob model=Qwen3-VL-30B
+    [STEP] step=1 action=click('123') reward=0.00 done=false error=null
+    [STEP] step=2 action=fill('456','text') reward=0.00 done=false error=null
+    [STEP] step=3 action=click('789') reward=1.00 done=true error=null
+    [END] success=true steps=3 score=1.00 rewards=0.00,0.00,1.00
 """
 
 import asyncio
@@ -26,40 +55,53 @@ from openai import OpenAI
 from client import CuratorEnv
 from models import CuratorAction
 
-IMAGE_NAME = os.getenv("IMAGE_NAME")
-API_KEY = os.getenv("HF_TOKEN") or os.getenv("API_KEY")
+HF_SPACE_URL = "https://huggingface.co/spaces/kgdrathan/openenv-curator"
+
+LOCAL_IMAGE_NAME = os.getenv("LOCAL_IMAGE_NAME") or os.getenv("IMAGE_NAME")
+API_KEY = "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"
+
+API_BASE_URL = "https://openrouter.ai/api/v1"
+MODEL_NAME = "nvidia/nemotron-3-nano-30b-a3b:free"
+
+API_KEY = os.getenv("API_KEY") or os.getenv("HF_TOKEN")
 
 API_BASE_URL = os.getenv("API_BASE_URL") or "https://router.huggingface.co/v1"
-MODEL_NAME = os.getenv("MODEL_NAME") or "meta-llama/Llama-3.2-1B-Instruct"
-TASK_NAME = os.getenv("CURATOR_TASK", "hard")
+MODEL_NAME = os.getenv("MODEL_NAME") or "Qwen/Qwen2.5-72B-Instruct"
+
+TASK_NAMES = os.getenv("CURATOR_TASKS", "easy,medium,hard").split(",")
 BENCHMARK = "curator"
 TEMPERATURE = 0.3
 MAX_TOKENS = 2000
 SUCCESS_SCORE_THRESHOLD = 0.3
 
+
 SYSTEM_PROMPT = textwrap.dedent("""
     You are a content curation agent. You help users find the most relevant
     articles from a pool of content items based on their interest profile.
 
-    Available actions (respond with valid JSON):
+    Available actions (respond with ONE JSON object, nothing else):
 
-    1. Filter (remove irrelevant items):
+    1. Filter — remove irrelevant items from the pool:
        {"action_type": "filter", "item_ids": ["id1", "id2", ...]}
 
-    2. Categorize items:
+    2. Categorize — tag items by priority:
        {"action_type": "categorize", "categories": {"id1": "urgent", "id2": "skip", ...}}
        Categories: "urgent", "read_later", "share", "skip"
 
-    3. Rank items by relevance:
+    3. Rank — order remaining items by relevance (best first):
        {"action_type": "rank", "rankings": ["best_id", "second_id", ...]}
 
-    4. Final recommendation (ends episode):
+    4. Recommend — final selection (ENDS the episode):
        {"action_type": "recommend", "item_ids": ["id1", "id2", ...]}
 
-    Strategy: First filter out clearly irrelevant items, then rank the remainder,
-    then recommend the top items.
+    STRATEGY — you MUST follow these steps in order across multiple turns:
+    Step 1: FILTER out clearly irrelevant items (low match to user interests).
+    Step 2: CATEGORIZE remaining items based on relevance to the user profile.
+    Step 3: RANK the remaining items by relevance (best first).
+    Step 4: Only RECOMMEND when you are confident in your top picks.
 
-    IMPORTANT: Respond with ONLY a JSON object, no markdown or explanation.
+    DO NOT use "recommend" until you have filtered, categorized, and ranked.
+    IMPORTANT: Respond with ONLY a single JSON object per turn. No markdown, no explanation.
 """).strip()
 
 
@@ -175,6 +217,7 @@ def get_model_action(
     client: OpenAI,
     obs: Any,
     step: int,
+    max_steps: int,
     last_feedback: Optional[str],
     messages: List[Dict[str, str]],
 ) -> Dict:
@@ -182,6 +225,8 @@ def get_model_action(
     user_prompt = build_user_prompt(obs, step, last_feedback)
     messages.append({"role": "user", "content": user_prompt})
 
+    has_prior_work = obs.task_info and (obs.task_info.items_filtered > 0 or obs.task_info.items_categorized > 0)
+
     try:
         completion = client.chat.completions.create(
             model=MODEL_NAME,
@@ -194,29 +239,39 @@ def get_model_action(
         messages.append({"role": "assistant", "content": text})
         action = parse_action_from_response(text)
         if action and "action_type" in action:
+            # Block recommend on step 1 with no prior work
+            if action["action_type"] == "recommend" and step == 1 and not has_prior_work:
+                item_ids = action.get("item_ids", [])
+                if item_ids:
+                    return {"action_type": "rank", "rankings": item_ids}
             return action
     except Exception as exc:
         print(f"[DEBUG] Model request failed: {exc}", flush=True)
 
-    # Fallback: recommend first N items from pool
+    # Fallback: use rank on early steps, recommend on last step
     item_ids = [item.id if hasattr(item, "id") else item["id"] for item in obs.items]
     k = obs.task_info.recommend_k if obs.task_info else 5
-    return {"action_type": "recommend", "item_ids": item_ids[:k]}
-
+    if step >= max_steps - 1 or has_prior_work:
+        return {"action_type": "recommend", "item_ids": item_ids[:k]}
+    return {"action_type": "rank", "rankings": item_ids[:k]}
 
-async def main() -> None:
-    llm_client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
 
-    async with CuratorEnv(base_url="http://localhost:8000") as env:
-        rewards: List[float] = []
-        steps_taken = 0
-        score = 0.0
-        success = False
-        last_feedback: Optional[str] = None
+async def run_episode(
+    env: Any,
+    client: OpenAI,
+    task_name: str,
+) -> None:
+    """Run one full episode for *task_name*, emitting [START] / [STEP]* / [END]."""
+    rewards: List[float] = []
+    steps_taken = 0
+    score = 0.0
+    success = False
+    last_feedback: Optional[str] = None
 
-        log_start(task=TASK_NAME, env=BENCHMARK, model=MODEL_NAME)
+    log_start(task=task_name, env=BENCHMARK, model=MODEL_NAME)
 
-        result = await env.reset(task_id=TASK_NAME)
+    try:
+        result = await env.reset(task_id=task_name)
         obs = result.observation
 
         task_info = obs.task_info
@@ -229,7 +284,7 @@ async def main() -> None:
             if result.done:
                 break
 
-            action_dict = get_model_action(llm_client, obs, step, last_feedback, messages)
+            action_dict = get_model_action(client, obs, step, max_steps, last_feedback, messages)
             action = CuratorAction(**action_dict)
 
             result = await env.step(action)
@@ -265,8 +320,30 @@ async def main() -> None:
         score = min(max(score, 0.0), 1.0)
         success = score >= SUCCESS_SCORE_THRESHOLD
 
+    finally:
         log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
 
 
+async def main() -> None:
+    client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
+
+    if LOCAL_IMAGE_NAME:
+        env = await CuratorEnv.from_docker_image(LOCAL_IMAGE_NAME)
+    else:
+        env = await CuratorEnv.from_env(
+            HF_SPACE_URL,
+            use_docker=False,
+        )
+
+    try:
+        for task_name in TASK_NAMES:
+            await run_episode(env, client, task_name)
+    finally:
+        try:
+            await env.close()
+        except Exception as e:
+            print(f"[DEBUG] env.close() error (cleanup): {e}", flush=True)
+
+
 if __name__ == "__main__":
     asyncio.run(main())

test_submission.sh

@@ -0,0 +1,315 @@
+#!/usr/bin/env bash
+#
+# test_submission.sh — Local end-to-end test for Curator Environment
+#
+# Starts the server, runs inference against all 3 tasks, validates output format,
+# and optionally runs pre-validation.sh.
+#
+# Usage:
+#   bash test_submission.sh
+#   bash test_submission.sh --skip-inference    # skip LLM inference (needs API key)
+#   bash test_submission.sh --skip-prevalidation # skip pre-validation.sh
+#
+# Required env vars for inference:
+#   HF_TOKEN or API_KEY — LLM API key
+#
+# Optional env vars:
+#   API_BASE_URL   — LLM endpoint (default: https://router.huggingface.co/v1)
+#   MODEL_NAME     — model to use (default: meta-llama/Llama-3.2-1B-Instruct)
+#   HF_SPACE_URL   — your HF Space URL for pre-validation (default: https://kgdrathan-openenv-curator.hf.space)
+
+set -uo pipefail
+
+REPO_DIR="$(cd "$(dirname "$0")" && pwd)"
+cd "$REPO_DIR"
+
+HF_SPACE_URL="${HF_SPACE_URL:-https://kgdrathan-openenv-curator.hf.space}"
+SERVER_PORT=8000
+SERVER_PID=""
+SKIP_INFERENCE=false
+SKIP_PREVALIDATION=false
+
+for arg in "$@"; do
+  case "$arg" in
+    --skip-inference) SKIP_INFERENCE=true ;;
+    --skip-prevalidation) SKIP_PREVALIDATION=true ;;
+  esac
+done
+
+# Colors
+if [ -t 1 ]; then
+  RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BOLD='\033[1m'; NC='\033[0m'
+else
+  RED=''; GREEN=''; YELLOW=''; BOLD=''; NC=''
+fi
+
+log()  { printf "[%s] %b\n" "$(date -u +%H:%M:%S)" "$*"; }
+pass() { log "${GREEN}PASS${NC} -- $1"; }
+fail() { log "${RED}FAIL${NC} -- $1"; }
+warn() { log "${YELLOW}WARN${NC} -- $1"; }
+
+cleanup() {
+  if [ -n "$SERVER_PID" ] && kill -0 "$SERVER_PID" 2>/dev/null; then
+    log "Stopping server (PID $SERVER_PID)..."
+    kill "$SERVER_PID" 2>/dev/null
+    wait "$SERVER_PID" 2>/dev/null
+  fi
+}
+trap cleanup EXIT
+
+PASS_COUNT=0
+FAIL_COUNT=0
+
+# =========================================================================
+printf "\n${BOLD}========================================${NC}\n"
+printf "${BOLD}  Curator Submission Test Suite${NC}\n"
+printf "${BOLD}========================================${NC}\n\n"
+
+# =========================================================================
+# Step 1: Check required files
+# =========================================================================
+log "${BOLD}Step 1/6: Checking required files${NC}"
+
+REQUIRED_FILES=(
+  "inference.py"
+  "client.py"
+  "models.py"
+  "openenv.yaml"
+  "pyproject.toml"
+  "uv.lock"
+  "server/app.py"
+  "server/curator_environment.py"
+  "server/grader.py"
+  "data/tasks.json"
+  "data/items.json"
+  "data/ground_truth.json"
+  "Dockerfile"
+)
+
+FILES_OK=true
+for f in "${REQUIRED_FILES[@]}"; do
+  if [ ! -f "$REPO_DIR/$f" ]; then
+    fail "Missing required file: $f"
+    FILES_OK=false
+    FAIL_COUNT=$((FAIL_COUNT + 1))
+  fi
+done
+if [ "$FILES_OK" = true ]; then
+  pass "All required files present (${#REQUIRED_FILES[@]} files)"
+  PASS_COUNT=$((PASS_COUNT + 1))
+fi
+
+# =========================================================================
+# Step 2: Check env vars
+# =========================================================================
+log "${BOLD}Step 2/6: Checking environment variables${NC}"
+
+API_KEY="${HF_TOKEN:-${API_KEY:-}}"
+if [ -z "$API_KEY" ]; then
+  warn "HF_TOKEN / API_KEY not set — inference step will be skipped"
+  SKIP_INFERENCE=true
+else
+  pass "API key is set"
+  PASS_COUNT=$((PASS_COUNT + 1))
+fi
+
+log "  API_BASE_URL = ${API_BASE_URL:-https://router.huggingface.co/v1 (default)}"
+log "  MODEL_NAME   = ${MODEL_NAME:-meta-llama/Llama-3.2-1B-Instruct (default)}"
+
+# =========================================================================
+# Step 3: openenv validate (local)
+# =========================================================================
+log "${BOLD}Step 3/6: Running openenv validate${NC}"
+
+if ! command -v openenv &>/dev/null && ! uv run openenv validate --help &>/dev/null 2>&1; then
+  warn "openenv CLI not found — skipping local validation"
+else
+  VALIDATE_CMD="openenv validate"
+  if ! command -v openenv &>/dev/null; then
+    VALIDATE_CMD="uv run openenv validate"
+  fi
+  VALIDATE_OUTPUT=$($VALIDATE_CMD 2>&1) && VALIDATE_OK=true || VALIDATE_OK=false
+
+  if [ "$VALIDATE_OK" = true ]; then
+    pass "openenv validate passed"
+    PASS_COUNT=$((PASS_COUNT + 1))
+  else
+    fail "openenv validate failed"
+    printf "%s\n" "$VALIDATE_OUTPUT"
+    FAIL_COUNT=$((FAIL_COUNT + 1))
+  fi
+fi
+
+# =========================================================================
+# Step 4: Start server
+# =========================================================================
+log "${BOLD}Step 4/6: Starting local server${NC}"
+
+# Check if port is already in use
+if curl -s -o /dev/null -w "%{http_code}" http://localhost:$SERVER_PORT/health --max-time 2 2>/dev/null | grep -q "200"; then
+  warn "Server already running on port $SERVER_PORT — using existing server"
+else
+  uv run uvicorn server.app:app --host 0.0.0.0 --port $SERVER_PORT &>"$REPO_DIR/.test_server.log" &
+  SERVER_PID=$!
+  log "  Server starting (PID $SERVER_PID)..."
+
+  # Wait for server to be ready
+  READY=false
+  for i in $(seq 1 30); do
+    if curl -s -o /dev/null -w "%{http_code}" http://localhost:$SERVER_PORT/health --max-time 2 2>/dev/null | grep -q "200"; then
+      READY=true
+      break
+    fi
+    sleep 1
+  done
+
+  if [ "$READY" = true ]; then
+    pass "Server is up and healthy"
+    PASS_COUNT=$((PASS_COUNT + 1))
+  else
+    fail "Server failed to start within 30s"
+    log "  Last 20 lines of server log:"
+    tail -20 "$REPO_DIR/.test_server.log" 2>/dev/null
+    FAIL_COUNT=$((FAIL_COUNT + 1))
+    printf "\n${RED}${BOLD}Cannot continue without server. Exiting.${NC}\n\n"
+    exit 1
+  fi
+fi
+
+# Quick endpoint checks
+log "  Checking endpoints..."
+for ENDPOINT in "/health" "/schema" "/metadata"; do
+  HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:$SERVER_PORT$ENDPOINT --max-time 5 2>/dev/null)
+  if [ "$HTTP_CODE" = "200" ]; then
+    log "    GET $ENDPOINT -> $HTTP_CODE OK"
+  else
+    warn "  GET $ENDPOINT -> $HTTP_CODE (expected 200)"
+  fi
+done
+
+RESET_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST -H "Content-Type: application/json" -d '{}' http://localhost:$SERVER_PORT/reset --max-time 10 2>/dev/null)
+if [ "$RESET_CODE" = "200" ]; then
+  log "    POST /reset -> $RESET_CODE OK"
+else
+  warn "  POST /reset -> $RESET_CODE (expected 200)"
+fi
+
+# =========================================================================
+# Step 5: Run inference
+# =========================================================================
+log "${BOLD}Step 5/6: Running inference (all 3 tasks)${NC}"
+
+if [ "$SKIP_INFERENCE" = true ]; then
+  warn "Inference skipped (no API key or --skip-inference)"
+else
+  INFERENCE_OUTPUT="$REPO_DIR/.test_inference_output.txt"
+  INFERENCE_EXIT=0
+  LOCAL_IMAGE_NAME="http://localhost:$SERVER_PORT" uv run python inference.py >"$INFERENCE_OUTPUT" 2>"$REPO_DIR/.test_inference_stderr.txt" || INFERENCE_EXIT=$?
+
+  if [ $INFERENCE_EXIT -ne 0 ]; then
+    fail "inference.py exited with code $INFERENCE_EXIT"
+    log "  stderr:"
+    tail -20 "$REPO_DIR/.test_inference_stderr.txt" 2>/dev/null
+    FAIL_COUNT=$((FAIL_COUNT + 1))
+  else
+    pass "inference.py exited successfully"
+    PASS_COUNT=$((PASS_COUNT + 1))
+  fi
+
+  # Validate output format
+  log "  Validating stdout format..."
+  printf "\n  --- inference.py stdout ---\n"
+  cat "$INFERENCE_OUTPUT"
+  printf "  --- end stdout ---\n\n"
+
+  START_COUNT=$(grep -c '^\[START\]' "$INFERENCE_OUTPUT" 2>/dev/null || echo 0)
+  END_COUNT=$(grep -c '^\[END\]' "$INFERENCE_OUTPUT" 2>/dev/null || echo 0)
+  STEP_COUNT=$(grep -c '^\[STEP\]' "$INFERENCE_OUTPUT" 2>/dev/null || echo 0)
+
+  log "  [START] lines: $START_COUNT (expected: 3)"
+  log "  [STEP]  lines: $STEP_COUNT"
+  log "  [END]   lines: $END_COUNT (expected: 3)"
+
+  if [ "$START_COUNT" -ge 3 ] && [ "$END_COUNT" -ge 3 ]; then
+    pass "All 3 tasks produced [START]/[END] blocks"
+    PASS_COUNT=$((PASS_COUNT + 1))
+  else
+    fail "Expected 3 [START] and 3 [END] lines, got $START_COUNT/$END_COUNT"
+    FAIL_COUNT=$((FAIL_COUNT + 1))
+  fi
+
+  # Check each [END] has score in [0,1]
+  SCORE_OK=true
+  while IFS= read -r line; do
+    SCORE=$(echo "$line" | grep -oE 'score=[0-9]+\.[0-9]+' | head -1 | cut -d= -f2)
+    if [ -n "$SCORE" ]; then
+      # Check score is between 0 and 1 (using awk for float comparison)
+      IN_RANGE=$(awk "BEGIN { print ($SCORE >= 0.0 && $SCORE <= 1.0) ? 1 : 0 }")
+      if [ "$IN_RANGE" != "1" ]; then
+        fail "Score $SCORE is not in [0, 1]: $line"
+        SCORE_OK=false
+      fi
+    fi
+  done < <(grep '^\[END\]' "$INFERENCE_OUTPUT" 2>/dev/null)
+
+  if [ "$SCORE_OK" = true ] && [ "$END_COUNT" -ge 3 ]; then
+    pass "All scores in [0, 1]"
+    PASS_COUNT=$((PASS_COUNT + 1))
+  fi
+
+  # Check tasks: easy, medium, hard
+  for TASK in easy medium hard; do
+    if grep -q "\[START\] task=$TASK " "$INFERENCE_OUTPUT" 2>/dev/null; then
+      log "    task=$TASK found"
+    else
+      fail "Missing [START] for task=$TASK"
+      FAIL_COUNT=$((FAIL_COUNT + 1))
+    fi
+  done
+
+  # Check no non-protocol lines on stdout
+  NON_PROTOCOL=$(grep -cvE '^\[(START|STEP|END)\]' "$INFERENCE_OUTPUT" 2>/dev/null || echo 0)
+  if [ "$NON_PROTOCOL" -gt 0 ]; then
+    warn "$NON_PROTOCOL non-protocol lines found on stdout (should only have [START]/[STEP]/[END])"
+  fi
+fi
+
+# =========================================================================
+# Step 6: Pre-validation script (optional)
+# =========================================================================
+log "${BOLD}Step 6/6: Running pre-validation.sh${NC}"
+
+if [ "$SKIP_PREVALIDATION" = true ]; then
+  warn "Pre-validation skipped (--skip-prevalidation)"
+elif [ ! -f "$REPO_DIR/pre-validation.sh" ]; then
+  warn "pre-validation.sh not found — skipping"
+else
+  # pre-validation.sh needs a live HF Space URL; use local server as fallback
+  PREVALIDATION_OUTPUT=$(bash "$REPO_DIR/pre-validation.sh" "http://localhost:$SERVER_PORT" "$REPO_DIR" 2>&1) && PREVAL_OK=true || PREVAL_OK=false
+
+  if [ "$PREVAL_OK" = true ]; then
+    pass "pre-validation.sh passed"
+    PASS_COUNT=$((PASS_COUNT + 1))
+  else
+    fail "pre-validation.sh failed"
+    printf "%s\n" "$PREVALIDATION_OUTPUT"
+    FAIL_COUNT=$((FAIL_COUNT + 1))
+  fi
+fi
+
+# =========================================================================
+# Summary
+# =========================================================================
+printf "\n${BOLD}========================================${NC}\n"
+TOTAL=$((PASS_COUNT + FAIL_COUNT))
+if [ "$FAIL_COUNT" -eq 0 ]; then
+  printf "${GREEN}${BOLD}  All $PASS_COUNT checks passed!${NC}\n"
+else
+  printf "${RED}${BOLD}  $FAIL_COUNT/$TOTAL checks failed.${NC}\n"
+fi
+printf "${BOLD}========================================${NC}\n\n"
+
+# Cleanup temp files
+rm -f "$REPO_DIR/.test_server.log" "$REPO_DIR/.test_inference_output.txt" "$REPO_DIR/.test_inference_stderr.txt"
+
+exit $FAIL_COUNT