Fix Phase 2 validation: Add missing graders for advanced_efficiency and expert_optimization tasks

__init__.py

@@ -16,6 +16,8 @@ from .task_graders import (
     task_1_basic_ram_reduction_grader,
     task_2_energy_optimization_grader,
     task_3_balanced_optimization_grader,
+    task_4_advanced_efficiency_grader,
+    task_5_expert_optimization_grader,
 )
 
 __all__ = [
@@ -30,4 +32,6 @@ __all__ = [
     "task_1_basic_ram_reduction_grader",
     "task_2_energy_optimization_grader",
     "task_3_balanced_optimization_grader",
+    "task_4_advanced_efficiency_grader",
+    "task_5_expert_optimization_grader",
 ]

graders.json

@@ -3,7 +3,7 @@
   "spec_version": "1.0",
   "type": "rl-environment",
   "real_world_application": "System resource optimization for data centers, cloud infrastructure, edge computing, and IoT devices",
-  "total_graders": 3,
+  "total_graders": 5,
   "minimum_required_graders": 3,
   "validation_status": "PASS",
   "scoring_scale": "0.0 (worst performance) to 1.0 (best performance)",
@@ -112,10 +112,62 @@
           "score": 0.925
         }
       }
+    },
+    {
+      "id": "task_4_advanced_efficiency_grader",
+      "name": "advanced_efficiency",
+      "display_name": "Advanced Efficiency",
+      "difficulty": 4,
+      "category": "hard",
+      "description": "Achieve RAM below 50% and energy below 4 kWh",
+      "targets": {
+        "ram_percentage": 50.0,
+        "energy_kwh": 4.0,
+        "max_steps": 25
+      },
+      "scoring_methodology": "Balance Score (90%: RAM Score 50% + Energy Score 50%) + Step Efficiency Bonus (10%)",
+      "real_world_application": "Highly constrained embedded systems and IoT devices",
+      "score_examples": {
+        "worst_case": {
+          "ram": 100.0,
+          "energy": 10.0,
+          "steps": 50,
+          "score": 0.0
+        },
+        "target_case": {
+          "ram": 50.0,
+          "energy": 4.0,
+          "steps": 25,
+          "score": 0.9
+        }
+      }
+    },
+    {
+      "id": "task_5_expert_optimization_grader",
+      "name": "expert_optimization",
+      "display_name": "Expert Optimization",
+      "difficulty": 5,
+      "category": "expert",
+      "description": "Master level: RAM below 40% and energy below 3 kWh",
+      "targets": {
+        "ram_percentage": 40.0,
+        "energy_kwh": 3.0,
+        "max_steps": 30
+      },
+      "scoring_methodology": "Balance Score (90%: RAM Score 60% + Energy Score 40%) + Step Efficiency Bonus (10%)",
+      "real_world_application": "Mission-critical space, deep-sea probes, and highly scaled edge clusters",
+      "score_examples": {
+        "worst_case": {
+          "ram": 100.0,
+          "energy": 10.0,
+          "steps": 50,
+          "score": 0.0
+        }
+      }
     }
   ],
   "summary": {
-    "graders_count": 3,
+    "graders_count": 5,
     "min_graders_required": 3,
     "graders_detected": true,
     "different_scores_returned": true,

graders_manifest.py

@@ -29,7 +29,7 @@ GRADERS_MANIFEST = {
     "environment_type": "OpenEnv RL Environment",
     "version": "1.0.0",
     "spec_version": "1",
-    "total_graders": 3,
+    "total_graders": 5,
     "minimum_required_graders": 3,
     "validation_requirement_met": True,  # 3 >= 3
     "real_world_application": "System resource optimization for production data centers, cloud infrastructure, and edge computing devices",
@@ -112,6 +112,46 @@ GRADERS_MANIFEST = {
                 "score_0_8_or_higher": {"scenario": "Good Performance", "ram": 60.0, "energy": 5.0, "steps": 18},
                 "score_0_9_or_higher": {"scenario": "Excellent Performance", "ram": 50.0, "energy": 4.0, "steps": 15}
             }
+        },
+        
+        "task_4_advanced_efficiency_grader": {
+            "task_name": "advanced_efficiency",
+            "display_name": "Task 4: Advanced Efficiency",
+            "difficulty_level": 4,
+            "difficulty_category": "HARD",
+            "description": "Agent must achieve RAM below 50% and energy below 4 kWh",
+            "targets": {
+                "ram_usage_percentage": 50.0,
+                "energy_consumption_kwh": 4.0,
+                "max_steps_allowed": 25
+            },
+            "scoring_methodology": {
+                "formula": "((ram_score * 0.5 + energy_score * 0.5) * 0.9) + step_bonus"
+            },
+            "real_world_context": "Highly constrained embedded systems and IoT devices.",
+            "performance_examples": {
+                "score_0_0": {"scenario": "Worst Performance", "ram": 100.0, "energy": 10.0, "steps": 50}
+            }
+        },
+        
+        "task_5_expert_optimization_grader": {
+            "task_name": "expert_optimization",
+            "display_name": "Task 5: Expert Optimization",
+            "difficulty_level": 5,
+            "difficulty_category": "EXPERT",
+            "description": "Master level: Agent must reduce RAM below 40% and energy below 3 kWh",
+            "targets": {
+                "ram_usage_percentage": 40.0,
+                "energy_consumption_kwh": 3.0,
+                "max_steps_allowed": 30
+            },
+            "scoring_methodology": {
+                "formula": "((ram_score * 0.6 + energy_score * 0.4) * 0.9) + step_bonus"
+            },
+            "real_world_context": "Mission-critical space, deep-sea probes, and highly scaled edge clusters.",
+            "performance_examples": {
+                "score_0_0": {"scenario": "Worst Performance", "ram": 100.0, "energy": 10.0, "steps": 50}
+            }
         }
     },
     
@@ -125,13 +165,14 @@ GRADERS_MANIFEST = {
     },
     
     "environment_stats": {
-        "total_difficulty_levels": 3,
+        "total_difficulty_levels": 5,
         "min_difficulty": 1,
-        "max_difficulty": 3,
+        "max_difficulty": 5,
         "task_distribution": {
             "easy": 1,
             "medium": 1,
-            "hard": 1
+            "hard": 2,
+            "expert": 1
         }
     }
 }

openenv_he_demo.egg-info/SOURCES.txt

@@ -1,6 +1,7 @@
 README.md
 __init__.py
 client.py
+graders.py
 gym_wrapper.py
 inference.py
 models.py
@@ -10,12 +11,16 @@ train_agent.py
 validate.py
 ./__init__.py
 ./client.py
+./graders.py
+./graders_manifest.py
 ./gym_wrapper.py
 ./inference.py
 ./models.py
+./task_graders.py
 ./test_environment.py
 ./train_agent.py
 ./validate.py
+./validate_comprehensive.py
 openenv_he_demo.egg-info/PKG-INFO
 openenv_he_demo.egg-info/SOURCES.txt
 openenv_he_demo.egg-info/dependency_links.txt

openenv_he_demo.egg-info/top_level.txt

@@ -1 +1,2 @@
+graders
 he_demo

task_graders.py

@@ -186,6 +186,70 @@ def task_3_balanced_optimization_grader(observation: EnergyOptimizationObservati
     return round(composite_score, 3)
 
 
+# ============================================================================
+# TASK 4: Advanced Efficiency (Hard Level - Difficulty 4)
+# ============================================================================
+
+def task_4_advanced_efficiency_grader(observation: EnergyOptimizationObservation) -> float:
+    """
+    Grade Task 4: Advanced Efficiency
+    
+    Target: Achieve RAM below 50% and energy below 4 kWh within 25 steps.
+    """
+    ram_target = 50.0
+    energy_target = 4.0
+    max_steps = 25
+    
+    ram_baseline = 100.0
+    energy_baseline = 10.0
+    
+    ram_score = max(0.0, min(1.0, (ram_baseline - observation.ram_usage) / (ram_baseline - ram_target)))
+    energy_score = max(0.0, min(1.0, (energy_baseline - observation.energy_consumption) / (energy_baseline - energy_target)))
+    
+    balance_score = (ram_score + energy_score) / 2.0
+    
+    if observation.steps_taken <= max_steps:
+        step_bonus = min(0.1, (max_steps - observation.steps_taken) / max_steps * 0.1)
+    else:
+        step_bonus = max(-0.2, -(observation.steps_taken - max_steps) * 0.05)
+        
+    composite_score = max(0.0, min(1.0, (balance_score * 0.9) + step_bonus))
+    
+    return round(composite_score, 3)
+
+
+# ============================================================================
+# TASK 5: Expert Optimization (Master Level - Difficulty 5)
+# ============================================================================
+
+def task_5_expert_optimization_grader(observation: EnergyOptimizationObservation) -> float:
+    """
+    Grade Task 5: Expert Optimization
+    
+    Target: Master level: RAM below 40% and energy below 3 kWh within 30 steps.
+    """
+    ram_target = 40.0
+    energy_target = 3.0
+    max_steps = 30
+    
+    ram_baseline = 100.0
+    energy_baseline = 10.0
+    
+    ram_score = max(0.0, min(1.0, (ram_baseline - observation.ram_usage) / (ram_baseline - ram_target)))
+    energy_score = max(0.0, min(1.0, (energy_baseline - observation.energy_consumption) / (energy_baseline - energy_target)))
+    
+    balance_score = (ram_score * 0.6) + (energy_score * 0.4)
+    
+    if observation.steps_taken <= max_steps:
+        step_bonus = min(0.1, (max_steps - observation.steps_taken) / max_steps * 0.1)
+    else:
+        step_bonus = max(-0.3, -(observation.steps_taken - max_steps) * 0.05)
+        
+    composite_score = max(0.0, min(1.0, (balance_score * 0.9) + step_bonus))
+    
+    return round(composite_score, 3)
+
+
 # ============================================================================
 # Registry and Metadata
 # ============================================================================
@@ -227,6 +291,30 @@ TASK_GRADERS: Dict[str, Dict[str, Any]] = {
         "max_steps": 20,
         "category": "hard",
         "real_world_application": "Production system optimization with dual constraints"
+    },
+    "advanced_efficiency": {
+        "grader": task_4_advanced_efficiency_grader,
+        "name": "advanced_efficiency",
+        "display_name": "Advanced Efficiency",
+        "difficulty": 4,
+        "description": "Achieve RAM below 50% and energy below 4 kWh",
+        "target_ram": 50.0,
+        "target_energy": 4.0,
+        "max_steps": 25,
+        "category": "hard",
+        "real_world_application": "Highly constrained embedded systems and IoT devices"
+    },
+    "expert_optimization": {
+        "grader": task_5_expert_optimization_grader,
+        "name": "expert_optimization",
+        "display_name": "Expert Optimization",
+        "difficulty": 5,
+        "description": "Master level: RAM below 40% and energy below 3 kWh",
+        "target_ram": 40.0,
+        "target_energy": 3.0,
+        "max_steps": 30,
+        "category": "expert",
+        "real_world_application": "Mission-critical space, deep-sea probes, and highly scaled edge clusters"
     }
 }
 

validate-submission.sh

@@ -0,0 +1,185 @@
+#!/usr/bin/env bash
+#
+# validate-submission.sh — OpenEnv Submission Validator
+#
+# Checks that your HF Space is live, Docker image builds, and openenv validate passes.
+#
+# Prerequisites:
+#   - Docker:       https://docs.docker.com/get-docker/
+#   - openenv-core: pip install openenv-core
+#   - curl (usually pre-installed)
+#
+# Run:
+#   curl -fsSL https://raw.githubusercontent.com/<owner>/<repo>/main/scripts/validate-submission.sh | bash -s -- <ping_url> [repo_dir]
+#
+#   Or download and run locally:
+#     chmod +x validate-submission.sh
+#     ./validate-submission.sh <ping_url> [repo_dir]
+#
+# Arguments:
+#   ping_url   Your HuggingFace Space URL (e.g. https://your-space.hf.space)
+#   repo_dir   Path to your repo (default: current directory)
+#
+# Examples:
+#   ./validate-submission.sh https://my-team.hf.space
+#   ./validate-submission.sh https://my-team.hf.space ./my-repo
+#
+
+set -uo pipefail
+
+DOCKER_BUILD_TIMEOUT=600
+if [ -t 1 ]; then
+  RED='\033[0;31m'
+  GREEN='\033[0;32m'
+  YELLOW='\033[1;33m'
+  BOLD='\033[1m'
+  NC='\033[0m'
+else
+  RED='' GREEN='' YELLOW='' BOLD='' NC=''
+fi
+
+run_with_timeout() {
+  local secs="$1"; shift
+  if command -v timeout >/dev/null; then
+    timeout "$secs" "$@"
+  elif command -v gtimeout >/dev/null; then
+    gtimeout "$secs" "$@"
+  else
+    "$@" &
+    local pid=$!
+    ( sleep "$secs" && kill "$pid" 2>/dev/null ) &
+    local watcher=$!
+    wait "$pid" 2>/dev/null
+    local rc=$?
+    kill "$watcher" 2>/dev/null
+    wait "$watcher" 2>/dev/null
+    return $rc
+  fi
+}
+
+portable_mktemp() {
+  local prefix="${1:-validate}"
+  mktemp "${TMPDIR:-/tmp}/${prefix}-XXXXXX" 2>/dev/null || mktemp
+}
+
+CLEANUP_FILES=()
+cleanup() { rm -f "${CLEANUP_FILES[@]+"${CLEANUP_FILES[@]}"}"; }
+trap cleanup EXIT
+
+PING_URL="${1:-}"
+REPO_DIR="${2:-.}"
+
+if [ -z "$PING_URL" ]; then
+  printf "Usage: %s <ping_url> [repo_dir]\n" "$0"
+  printf "\n"
+  printf "  ping_url   Your HuggingFace Space URL (e.g. https://your-space.hf.space)\n"
+  printf "  repo_dir   Path to your repo (default: current directory)\n"
+  exit 1
+fi
+
+if ! REPO_DIR="$(cd "$REPO_DIR" 2>/dev/null && pwd)"; then
+  printf "Error: directory '%s' not found\n" "${2:-.}"
+  exit 1
+fi
+PING_URL="${PING_URL%/}"
+export PING_URL
+PASS=0
+
+log()  { printf "[%s] %b\n" "$(date -u +%H:%M:%S)" "$*"; }
+pass() { log "${GREEN}PASSED${NC} -- $1"; PASS=$((PASS + 1)); }
+fail() { log "${RED}FAILED${NC} -- $1"; }
+hint() { printf "  ${YELLOW}Hint:${NC} %b\n" "$1"; }
+stop_at() {
+  printf "\n"
+  printf "${RED}${BOLD}Validation stopped at %s.${NC} Fix the above before continuing.\n" "$1"
+  exit 1
+}
+
+printf "\n"
+printf "${BOLD}========================================${NC}\n"
+printf "${BOLD}  OpenEnv Submission Validator${NC}\n"
+printf "${BOLD}========================================${NC}\n"
+log "Repo:     $REPO_DIR"
+log "Ping URL: $PING_URL"
+printf "\n"
+
+log "${BOLD}Step 1/3: Pinging HF Space${NC} ($PING_URL/reset) ..."
+
+CURL_OUTPUT=$(portable_mktemp "validate-curl")
+CLEANUP_FILES+=("$CURL_OUTPUT")
+HTTP_CODE=$(curl -s -o "$CURL_OUTPUT" -w "%{http_code}" -X POST \
+  -H "Content-Type: application/json" -d '{}' \
+  "$PING_URL/reset" --max-time 30 2>"$CURL_OUTPUT" || printf "000")
+
+if [ "$HTTP_CODE" = "200" ]; then
+  pass "HF Space is live and responds to /reset"
+elif [ "$HTTP_CODE" = "000" ]; then
+  fail "HF Space not reachable (connection failed or timed out)"
+  hint "Check your network connection and that the Space is running."
+  hint "Try: curl -s -o /dev/null -w '%{http_code}' -X POST $PING_URL/reset"
+  stop_at "Step 1"
+else
+  fail "HF Space /reset returned HTTP $HTTP_CODE (expected 200)"
+  hint "Make sure your Space is running and the URL is correct."
+  hint "Try opening $PING_URL in your browser first."
+  stop_at "Step 1"
+fi
+
+log "${BOLD}Step 2/3: Running docker build${NC} ..."
+
+if ! command -v docker >/dev/null; then
+  fail "docker command not found"
+  hint "Install Docker: https://docs.docker.com/get-docker/"
+  stop_at "Step 2"
+fi
+
+if [ -f "$REPO_DIR/Dockerfile" ]; then
+  DOCKER_CONTEXT="$REPO_DIR"
+elif [ -f "$REPO_DIR/server/Dockerfile" ]; then
+  DOCKER_CONTEXT="$REPO_DIR/server"
+else
+  fail "No Dockerfile found in repo root or server/ directory"
+  stop_at "Step 2"
+fi
+
+log "  Found Dockerfile in $DOCKER_CONTEXT"
+
+BUILD_OK=false
+BUILD_OUTPUT=$(run_with_timeout "$DOCKER_BUILD_TIMEOUT" docker build "$DOCKER_CONTEXT" 2>&1) && BUILD_OK=true
+
+if [ "$BUILD_OK" = true ]; then
+  pass "Docker build succeeded"
+else
+  fail "Docker build failed (timeout=${DOCKER_BUILD_TIMEOUT}s)"
+  printf "%s\n" "$BUILD_OUTPUT" | tail -20
+  stop_at "Step 2"
+fi
+
+log "${BOLD}Step 3/3: Running openenv validate${NC} ..."
+
+if ! command -v openenv >/dev/null; then
+  fail "openenv command not found"
+  hint "Install it: pip install openenv-core"
+  stop_at "Step 3"
+fi
+
+VALIDATE_OK=false
+VALIDATE_OUTPUT=$(cd "$REPO_DIR" && openenv validate 2>&1) && VALIDATE_OK=true
+
+if [ "$VALIDATE_OK" = true ]; then
+  pass "openenv validate passed"
+  [ -n "$VALIDATE_OUTPUT" ] && log "  $VALIDATE_OUTPUT"
+else
+  fail "openenv validate failed"
+  printf "%s\n" "$VALIDATE_OUTPUT"
+  stop_at "Step 3"
+fi
+
+printf "\n"
+printf "${BOLD}========================================${NC}\n"
+printf "${GREEN}${BOLD}  All 3/3 checks passed!${NC}\n"
+printf "${GREEN}${BOLD}  Your submission is ready to submit.${NC}\n"
+printf "${BOLD}========================================${NC}\n"
+printf "\n"
+
+exit 0

validate_comprehensive.py

@@ -115,10 +115,16 @@ def main():
     # ========================================================================
     # 4. TEST ALL GRADERS WITH MULTIPLE SCENARIOS
     # ========================================================================
-    print("\n[4] Testing All 3 Graders with Performance Scenarios")
+    print("\n[4] Testing All 5 Graders with Performance Scenarios")
     print("-" * 90)
     
-    all_task_names = ["basic_ram_reduction", "energy_optimization", "balanced_optimization"]
+    all_task_names = [
+        "basic_ram_reduction", 
+        "energy_optimization", 
+        "balanced_optimization",
+        "advanced_efficiency",
+        "expert_optimization"
+    ]
     
     for task_name in all_task_names:
         metadata = get_grader_metadata(task_name)