Finalize submission: Replace local validate.sh with comprehensive validate-submission.sh

OPENENV_SUBMISSION_CHECKLIST.md

@@ -306,9 +306,9 @@ TASK=hard   python inference.py   # expected: score < 0.8
 
 | Task | Difficulty | Model | Score | Steps | Notes |
 |------|-----------|-------|-------|-------|-------|
-| python-off-by-one | easy | Llama-3.3-70B-Instruct | 0.68 | 1 | |
-| js-auth-privilege | medium | Llama-3.3-70B-Instruct | 0.70 | 1 | |
-| python-sql-injection | hard | Llama-3.3-70B-Instruct | 0.54 | 1 | |
+| python-off-by-one | easy | Llama-3.3-70B-Instruct | 0.883 | 2 | |
+| js-idor-auth | medium | Llama-3.3-70B-Instruct | 0.500 | 2 | |
+| python-pickle-deserialization | hard | Llama-3.3-70B-Instruct | 0.512 | 2 | |
 
 - [x] The table is filled in with real numbers from a completed inference run.
 - [x] The easy task score is ≥ 0.6.
@@ -423,7 +423,7 @@ done
 
 Expected: Three complete runs, each emitting `[START]`, N×`[STEP]`, and `[END]` with no Python exceptions.
 
-- [x] ✓ PASSED — Easy score: 0.68 Medium score: 0.70 Hard score: 0.54
+- [x] ✓ PASSED — Easy score: 0.883 Medium score: 0.500 Hard score: 0.512
 
 ### Step 5 — Verify log format
 
@@ -514,10 +514,10 @@ When all items above are checked, fill in this block and attach it to your submi
 Environment Name:  Code Security Review
 HF Space URL:      https://huggingface.co/spaces/inmodel/code-review-env
 Baseline Scores:
-  - Easy task:     0.68  (task name: python-off-by-one)
-  - Medium task:   0.10  (task name: js-auth-privilege)
-  - Hard task:     0.75  (task name: python-sql-injection)
-Inference runtime: < 1 minute
+  - Easy task:     0.883 (task name: python-off-by-one)
+  - Medium task:   0.500 (task name: js-idor-auth)
+  - Hard task:     0.512 (task name: python-pickle-deserialization)
+Inference runtime: < 20 minutes
 Docker image size: 250 MB
 Submitted by:      NitishKumar
 Date:              2026-04-08

validate-submission.sh

@@ -0,0 +1,183 @@
+#!/usr/bin/env bash
+#
+# validate-submission.sh — OpenEnv Submission Validator
+#
+# Checks that your HF Space is live, Docker image builds, and openenv validate passes.
+#
+# Prerequisites:
+#   - Docker:       https://docs.docker.com/get-docker/
+#   - openenv-core: pip install openenv-core
+#   - curl (usually pre-installed)
+#
+# Run:
+#   curl -fsSL https://raw.githubusercontent.com/<owner>/<repo>/main/scripts/validate-submission.sh | bash -s -- <ping_url> [repo_dir]
+#
+#   Or download and run locally:
+#     chmod +x validate-submission.sh
+#     ./validate-submission.sh <ping_url> [repo_dir]
+#
+# Arguments:
+#   ping_url   Your HuggingFace Space URL (e.g. https://your-space.hf.space)
+#   repo_dir   Path to your repo (default: current directory)
+#
+# Examples:
+#   ./validate-submission.sh https://my-team.hf.space
+#   ./validate-submission.sh https://my-team.hf.space ./my-repo
+#
+
+set -uo pipefail
+
+DOCKER_BUILD_TIMEOUT=600
+if [ -t 1 ]; then
+  RED='\033[0;31m'
+  GREEN='\033[0;32m'
+  YELLOW='\033[1;33m'
+  BOLD='\033[1m'
+  NC='\033[0m'
+else
+  RED='' GREEN='' YELLOW='' BOLD='' NC=''
+fi
+
+run_with_timeout() {
+  local secs="$1"; shift
+  if command -v timeout &>/dev/null; then
+    timeout "$secs" "$@"
+  elif command -v gtimeout &>/dev/null; then
+    gtimeout "$secs" "$@"
+  else
+    "$@" &
+    local pid=$!
+    ( sleep "$secs" && kill "$pid" 2>/dev/null ) &
+    local watcher=$!
+    wait "$pid" 2>/dev/null
+    local rc=$?
+    kill "$watcher" 2>/dev/null
+    wait "$watcher" 2>/dev/null
+    return $rc
+  fi
+}
+
+portable_mktemp() {
+  local prefix="${1:-validate}"
+  mktemp "${TMPDIR:-/tmp}/${prefix}-XXXXXX" 2>/dev/null || mktemp
+}
+
+CLEANUP_FILES=()
+cleanup() { rm -f "${CLEANUP_FILES[@]+"${CLEANUP_FILES[@]}"}"; }
+trap cleanup EXIT
+
+PING_URL="${1:-}"
+REPO_DIR="${2:-.}"
+
+if [ -z "$PING_URL" ]; then
+  printf "Usage: %s <ping_url> [repo_dir]\n" "$0"
+  printf "\n"
+  printf "  ping_url   Your HuggingFace Space URL (e.g. https://your-space.hf.space)\n"
+  printf "  repo_dir   Path to your repo (default: current directory)\n"
+  exit 1
+fi
+
+if ! REPO_DIR="$(cd "$REPO_DIR" 2>/dev/null && pwd)"; then
+  printf "Error: directory '%s' not found\n" "${2:-.}"
+  exit 1
+fi
+PING_URL="${PING_URL%/}"
+export PING_URL
+PASS=0
+
+log()  { printf "[%s] %b\n" "$(date -u +%H:%M:%S)" "$*"; }
+pass() { log "${GREEN}PASSED${NC} -- $1"; PASS=$((PASS + 1)); }
+fail() { log "${RED}FAILED${NC} -- $1"; }
+hint() { printf "  ${YELLOW}Hint:${NC} %b\n" "$1"; }
+stop_at() {
+  printf "\n"
+  printf "${RED}${BOLD}Validation stopped at %s.${NC} Fix the above before continuing.\n" "$1"
+  exit 1
+}
+
+printf "\n"
+printf "${BOLD}========================================${NC}\n"
+printf "${BOLD}  OpenEnv Submission Validator${NC}\n"
+printf "${BOLD}========================================${NC}\n"
+log "Repo:     $REPO_DIR"
+log "Ping URL: $PING_URL"
+printf "\n"
+
+log "${BOLD}Step 1/3: Pinging HF Space${NC} ($PING_URL/reset) ..."
+
+CURL_OUTPUT=$(portable_mktemp "validate-curl")
+CLEANUP_FILES+=("$CURL_OUTPUT")
+HTTP_CODE=$(curl -s -o "$CURL_OUTPUT" -w "%{http_code}" -X POST \
+  -H "Content-Type: application/json" -d '{}' \
+  "$PING_URL/reset" --max-time 30 2>"$CURL_OUTPUT" || printf "000")
+
+if [ "$HTTP_CODE" = "200" ]; then
+  pass "HF Space is live and responds to /reset"
+elif [ "$HTTP_CODE" = "000" ]; then
+  fail "HF Space not reachable (connection failed or timed out)"
+  hint "Check your network connection and that the Space is running."
+  hint "Try: curl -s -o /dev/null -w '%%{http_code}' -X POST $PING_URL/reset"
+  stop_at "Step 1"
+else
+  fail "HF Space /reset returned HTTP $HTTP_CODE (expected 200)"
+  hint "Make sure your Space is running and the URL is correct."
+  hint "Try opening $PING_URL in your browser first."
+  stop_at "Step 1"
+fi
+
+log "${BOLD}Step 2/3: Running docker build${NC} ..."
+
+if ! command -v docker &>/dev/null; then
+  fail "docker command not found"
+  hint "Install Docker: https://docs.docker.com/get-docker/"
+  stop_at "Step 2"
+fi
+
+if [ -f "$REPO_DIR/Dockerfile" ]; then
+  DOCKER_CONTEXT="$REPO_DIR"
+elif [ -f "$REPO_DIR/server/Dockerfile" ]; then
+  DOCKER_CONTEXT="$REPO_DIR/server"
+else
+  fail "No Dockerfile found in repo root or server/ directory"
+  stop_at "Step 2"
+fi
+
+log "  Found Dockerfile in $DOCKER_CONTEXT"
+
+BUILD_OK=false
+BUILD_OUTPUT=$(run_with_timeout "$DOCKER_BUILD_TIMEOUT" docker build "$DOCKER_CONTEXT" 2>&1) && BUILD_OK=true
+
+if [ "$BUILD_OK" = true ]; then
+  pass "Docker build succeeded"
+else
+  fail "Docker build failed (timeout=${DOCKER_BUILD_TIMEOUT}s)"
+  printf "%s\n" "$BUILD_OUTPUT" | tail -20
+  stop_at "Step 2"
+fi
+
+log "${BOLD}Step 3/3: Running openenv validate${NC} ..."
+
+if ! command -v openenv &>/dev/null; then
+  fail "openenv command not found"
+  hint "Install it: pip install openenv-core"
+  stop_at "Step 3"
+fi
+
+VALIDATE_OK=false
+VALIDATE_OUTPUT=$(cd "$REPO_DIR" && openenv validate 2>&1) && VALIDATE_OK=true
+
+if [ "$VALIDATE_OK" = true ]; then
+  pass "openenv validate passed"
+  [ -n "$VALIDATE_OUTPUT" ] && log "  $VALIDATE_OUTPUT"
+else
+  fail "openenv validate failed"
+  printf "%s\n" "$VALIDATE_OUTPUT"
+  stop_at "Step 3"
+fi
+
+printf "\n"
+printf "${BOLD}========================================${NC}\n"
+printf "${GREEN}${BOLD}  All 3/3 checks passed!${NC}\n"
+printf "${GREEN}${BOLD}  Your submission is ready to submit.${NC}\n"
+printf "${BOLD}========================================${NC}\n"
+printf "\n"

validate.sh

@@ -1,103 +0,0 @@
-#!/bin/bash
-
-# OpenEnv Submission Validation Script
-
-set -e
-echo "═══════════════════════════════════════"
-echo "  OpenEnv Pre-Submission Validation"
-echo "═══════════════════════════════════════"
-echo ""
-
-# 1. Check for required root files
-echo "── 1. Required Files ──"
-FILES=("openenv.yaml" "inference.py" "README.md" "Dockerfile" "requirements.txt")
-for file in "${FILES[@]}"; do
-    if [ -f "$file" ]; then
-        echo "  ✅ $file"
-    else
-        echo "  ❌ Missing $file"
-        exit 1
-    fi
-done
-echo ""
-
-# 2. Check server/ module structure
-echo "── 2. Server Module Structure ──"
-SERVER_FILES=("server/__init__.py" "server/app.py" "server/models.py" "server/environment.py" "server/tasks.py" "server/grader.py")
-for file in "${SERVER_FILES[@]}"; do
-    if [ -f "$file" ]; then
-        echo "  ✅ $file"
-    else
-        echo "  ❌ Missing $file"
-        exit 1
-    fi
-done
-echo ""
-
-# 3. Activate venv & validate Python imports
-echo "── 3. Python Import Validation ──"
-source venv/bin/activate
-python3 -c "
-from server.tasks import TASKS
-from server.grader import grade_action
-from server.environment import CodeSecurityEnv
-from server.models import CodeReviewAction, CodeObservation, StepResult, StateResponse, ResetResponse, TaskInfo
-
-assert len(TASKS) >= 3, f'Expected 3+ tasks, got {len(TASKS)}'
-print('  ✅ All imports resolve correctly')
-print(f'     Tasks: {list(TASKS.keys())}')
-" || { echo "  ❌ Python import validation failed"; exit 1; }
-echo ""
-
-# 4. Quick grader smoke test
-echo "── 4. Grader Smoke Test ──"
-python3 -c "
-from server.environment import CodeSecurityEnv
-from server.models import Action
-
-env = CodeSecurityEnv()
-obs = env.reset('python-off-by-one')
-result = env.step(Action(**{
-    'bug_identified': True,
-    'bug_location': 'range(len(transactions) + 1)',
-    'bug_type': 'logic-error',
-    'bug_description': 'Off-by-one index error — the range goes one past the end causing an out of bounds IndexError',
-    'severity': 'medium',
-    'suggested_fix': 'Use range(len(transactions)) to fix the boundary',
-}))
-assert 0.0 <= result.reward <= 1.0, f'Reward out of range: {result.reward}'
-assert result.done is True
-print(f'  ✅ Grader returned reward={result.reward:.4f}, done={result.done}')
-
-# Verify zero-reward path
-env2 = CodeSecurityEnv()
-env2.reset('python-off-by-one')
-r2 = env2.step(Action(**{
-    'bug_identified': False,
-    'bug_location': '',
-    'bug_type': 'none',
-    'bug_description': 'No bug found',
-    'severity': 'none',
-    'suggested_fix': '',
-}))
-assert r2.reward == 0.0, f'Expected 0.0 for no-bug, got {r2.reward}'
-print(f'  ✅ No-bug path returns reward=0.0')
-" || { echo "  ❌ Grader smoke test failed"; exit 1; }
-echo ""
-
-# 5. Validate openenv.yaml
-echo "── 5. openenv.yaml Validation ──"
-python3 -c "
-import yaml
-with open('openenv.yaml', 'r') as f:
-    data = yaml.safe_load(f)
-assert 'name' in data, 'Missing name field'
-assert 'tasks' in data, 'Missing tasks field'
-assert len(data['tasks']) >= 3, f'Need 3+ tasks, got {len(data[\"tasks\"])}'
-print(f'  ✅ Valid YAML with {len(data[\"tasks\"])} tasks')
-" || { echo "  ❌ openenv.yaml validation failed"; exit 1; }
-echo ""
-
-echo "═══════════════════════════════════════"
-echo "  ✅ All checks passed!"
-echo "═══════════════════════════════════════"