Upload 36 files

api/server.py

@@ -7,9 +7,9 @@ Endpoints:
 
 from __future__ import annotations
 
-from typing import Any
+from typing import Any, Optional
 
-from fastapi import FastAPI, HTTPException
+from fastapi import Body, FastAPI, HTTPException, Request
 from pydantic import BaseModel, Field
 
 from kaggle_sim_env.environment import KaggleSimEnv
@@ -78,9 +78,14 @@ class ActionCategoryEntry(BaseModel):
 # ---------------------------------------------------------------------------
 
 @app.post("/reset", response_model=Observation)
-def reset(req: ResetRequest) -> Observation:
+async def reset(request: Request) -> Observation:
     try:
-        return env.reset(task_id=req.task_id)
+        body = await request.json()
+        task_id = body.get("task_id", "easy_churn")
+    except Exception:
+        task_id = "easy_churn"
+    try:
+        return env.reset(task_id=task_id)
     except ValueError as exc:
         raise HTTPException(status_code=400, detail=str(exc))
 
@@ -307,4 +312,37 @@ def _baseline_plan(task_id: str) -> list[Action]:
 
 @app.get("/health")
 def health() -> dict[str, str]:
-    return {"status": "ok"}
+    """OpenEnv runtime check expects status == 'healthy'."""
+    return {"status": "healthy"}
+
+
+@app.get("/metadata")
+def metadata() -> dict[str, str]:
+    """OpenEnv standard metadata endpoint."""
+    return {
+        "name": "KaggleSimEnv",
+        "description": (
+            "RL environment simulating Kaggle competitions with hierarchical actions, "
+            "causal dataset properties, failure-mode traps, and contextual scoring."
+        ),
+    }
+
+
+@app.get("/schema")
+def schema_endpoint() -> dict[str, Any]:
+    """OpenEnv combined JSON Schema for action, observation, and state."""
+    return {
+        "action": Action.model_json_schema(),
+        "observation": Observation.model_json_schema(),
+        "state": EnvState.model_json_schema(),
+    }
+
+
+@app.post("/mcp")
+def mcp_stub(payload: dict[str, Any] = Body(default_factory=dict)) -> dict[str, Any]:
+    """Minimal JSON-RPC envelope for OpenEnv runtime validation."""
+    return {
+        "jsonrpc": "2.0",
+        "id": payload.get("id"),
+        "result": {"ok": True},
+    }

baseline/__init__.py

@@ -0,0 +1 @@
+"""Baseline agent package."""

inference.py

@@ -0,0 +1,192 @@
+"""
+Inference script — KaggleSimEnv (submission entrypoint)
+======================================================
+
+MANDATORY (environment configuration)
+-------------------------------------
+- API_BASE_URL   : LLM API base URL (e.g. https://router.huggingface.co/v1)
+- MODEL_NAME     : Model id for chat completions
+- HF_TOKEN       : Hugging Face / API key (passed to OpenAI client as api_key)
+
+Optional
+--------
+- ENV_URL        : Base URL of the running KaggleSimEnv Space or server (default: http://127.0.0.1:7860)
+
+Requirements
+------------
+- Named ``inference.py`` in the project root
+- All LLM calls go through ``openai.OpenAI`` using the variables above
+- Completes all tasks from ``GET /tasks``, runs each episode, then ``POST /grader``; prints scores in [0.0, 1.0]
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+from typing import Any
+
+import requests
+from openai import OpenAI
+
+from baseline.run_baseline import SYSTEM_PROMPT, build_user_message, parse_llm_action
+from kaggle_sim_env.models import Action
+
+# Tunables (keep total runtime modest on 2 vCPU / 8GB)
+REQUEST_TIMEOUT_S = 60
+MAX_LLM_STEPS_PER_TASK = 14
+TEMPERATURE = 0.0
+MAX_TOKENS = 256
+
+
+def _require_env(name: str) -> str:
+    v = os.getenv(name, "").strip()
+    if not v:
+        print(f"Error: required environment variable {name} is not set.", file=sys.stderr)
+        sys.exit(1)
+    return v
+
+
+def _client() -> OpenAI:
+    base_url = _require_env("API_BASE_URL")
+    _require_env("MODEL_NAME")
+    # Spec: HF_TOKEN; allow OPENAI_API_KEY for local OpenAI-only runs
+    api_key = os.getenv("HF_TOKEN", "").strip() or os.getenv("OPENAI_API_KEY", "").strip()
+    if not api_key:
+        print(
+            "Error: set HF_TOKEN (or OPENAI_API_KEY for local OpenAI).",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+    return OpenAI(base_url=base_url, api_key=api_key)
+
+
+def _env_base() -> str:
+    return os.getenv("ENV_URL", "http://127.0.0.1:7860").rstrip("/")
+
+
+def list_task_ids(base: str) -> list[str]:
+    r = requests.get(f"{base}/tasks", timeout=REQUEST_TIMEOUT_S)
+    r.raise_for_status()
+    data = r.json()
+    return [t["task_id"] for t in data]
+
+
+def coerce_action_for_step(raw: Any) -> dict[str, Any]:
+    """Normalize LLM output so POST /step matches api.server StepRequest + Action."""
+    fallback = {"action_type": "submit", "parameters": {}}
+    if not isinstance(raw, dict):
+        return fallback
+    d = raw
+    if "action" in d and isinstance(d["action"], dict):
+        d = d["action"]
+    at = d.get("action_type")
+    params = d.get("parameters")
+    if params is None:
+        params = {}
+    if isinstance(params, str):
+        try:
+            params = json.loads(params)
+        except json.JSONDecodeError:
+            params = {}
+    if not isinstance(params, dict):
+        params = {}
+    if not at or not isinstance(at, str):
+        return fallback
+    at = at.strip()
+    try:
+        Action(action_type=at, parameters=params)
+    except Exception:
+        return fallback
+    return {"action_type": at, "parameters": params}
+
+
+def run_episode(client: OpenAI, model: str, base: str, task_id: str) -> dict[str, Any]:
+    r = requests.post(f"{base}/reset", json={"task_id": task_id}, timeout=REQUEST_TIMEOUT_S)
+    r.raise_for_status()
+    obs_dict: dict[str, Any] = r.json()
+
+    messages: list[dict[str, str]] = [{"role": "system", "content": SYSTEM_PROMPT}]
+    steps = 0
+
+    while not obs_dict.get("done", False) and steps < MAX_LLM_STEPS_PER_TASK:
+        messages.append({"role": "user", "content": build_user_message(obs_dict)})
+        response = client.chat.completions.create(
+            model=model,
+            messages=messages,
+            temperature=TEMPERATURE,
+            max_tokens=MAX_TOKENS,
+        )
+        raw = response.choices[0].message.content or "{}"
+        messages.append({"role": "assistant", "content": raw})
+
+        try:
+            parsed = parse_llm_action(raw)
+        except Exception:
+            parsed = {"action_type": "submit", "parameters": {}}
+
+        action_dict = coerce_action_for_step(parsed)
+
+        r = requests.post(f"{base}/step", json=action_dict, timeout=REQUEST_TIMEOUT_S)
+        if not r.ok:
+            detail = (r.text or "")[:1000]
+            raise RuntimeError(f"POST /step HTTP {r.status_code}: {detail}")
+        step_data = r.json()
+        obs_dict = step_data["observation"]
+        steps += 1
+
+    r = requests.post(f"{base}/grader", timeout=REQUEST_TIMEOUT_S)
+    r.raise_for_status()
+    return r.json()
+
+
+def _assert_score_range(grade: dict[str, Any], task_id: str) -> None:
+    for key in (
+        "final_score",
+        "performance_score",
+        "strategy_score",
+        "combo_score",
+        "trap_score",
+    ):
+        v = float(grade[key])
+        if not 0.0 <= v <= 1.0:
+            raise ValueError(f"{task_id}: {key}={v} not in [0, 1]")
+
+
+def main() -> None:
+    client = _client()
+    model = _require_env("MODEL_NAME")
+    base = _env_base()
+
+    task_ids = list_task_ids(base)
+    if len(task_ids) < 3:
+        raise RuntimeError(f"Expected at least 3 tasks from /tasks, got {len(task_ids)}")
+
+    print(f"ENV_URL={base}")
+    print(f"Tasks: {task_ids}\n")
+
+    all_grades: list[dict[str, Any]] = []
+    for tid in task_ids:
+        print("=" * 60)
+        print(f"Task: {tid}")
+        print("=" * 60)
+        grade = run_episode(client, model, base, tid)
+        _assert_score_range(grade, tid)
+        all_grades.append(grade)
+        print(
+            f"  final={grade['final_score']:.4f}  perf={grade['performance_score']:.4f}  "
+            f"strat={grade['strategy_score']:.4f}  combo={grade['combo_score']:.4f}  "
+            f"trap={grade['trap_score']:.4f}"
+        )
+
+    print("\n" + "=" * 60)
+    print("SUMMARY (all scores in [0.0, 1.0])")
+    print("=" * 60)
+    for g in all_grades:
+        print(f"  {g['task_id']:<22} final={float(g['final_score']):.4f}")
+    avg = sum(float(g["final_score"]) for g in all_grades) / len(all_grades)
+    print(f"\n  Mean final score: {avg:.4f}")
+
+
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -0,0 +1,22 @@
+[project]
+name = "kaggle-sim-env"
+version = "3.0.0"
+description = "Kaggle Simulation Environment for OpenEnv – RL environment simulating Kaggle competitions"
+authors = [{ name = "Aadi Gupta" }]
+requires-python = ">=3.10"
+
+dependencies = [
+    "fastapi>=0.115.0",
+    "uvicorn[standard]>=0.32.0",
+    "pydantic>=2.9.0",
+    "requests>=2.32.0",
+    "openai>=1.55.0",
+    "openenv-core>=0.2.0",
+]
+
+[project.scripts]
+server = "server.app:main"
+
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"

requirements.txt

@@ -3,3 +3,4 @@ uvicorn[standard]>=0.32.0
 pydantic>=2.9.0
 openai>=1.55.0
 requests>=2.32.0
+openenv-core>=0.2.0

server/app.py

@@ -0,0 +1,261 @@
+"""
+OpenEnv-compatible server for KaggleSimEnv.
+
+Wraps the KaggleSimEnv in the openenv Environment interface and exposes
+it via ``create_app``.  Additional custom endpoints (tasks, grader,
+baseline, actions) are mounted on the same FastAPI app.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from pydantic import Field
+
+# Ensure project root is importable
+_project_root = str(Path(__file__).resolve().parent.parent)
+if _project_root not in sys.path:
+    sys.path.insert(0, _project_root)
+
+from openenv.core.env_server.http_server import create_app
+from openenv.core.env_server.interfaces import Environment
+from openenv.core.env_server.types import (
+    Action as OEAction,
+    EnvironmentMetadata,
+    Observation as OEObservation,
+    State as OEState,
+)
+
+from kaggle_sim_env.environment import KaggleSimEnv
+from kaggle_sim_env.grader import GradeResult, Grader
+from kaggle_sim_env.models import (
+    Action as KSAction,
+    ActionType,
+    CATEGORY_MAP,
+    get_categories_for_action,
+)
+from kaggle_sim_env.tasks import TASK_REGISTRY, get_task
+
+
+# ── OpenEnv-compatible Pydantic models ───────────────────────────────────
+
+class KaggleAction(OEAction):
+    action_type: str
+    parameters: Dict[str, Any] = Field(default_factory=dict)
+
+
+class KaggleObservation(OEObservation):
+    dataset_metadata: Dict[str, Any] = Field(default_factory=dict)
+    applied_strategies: List[str] = Field(default_factory=list)
+    current_cv_score: float = 0.0
+    leaderboard_rank: int = 0
+    step_count: int = 0
+    max_steps: int = 10
+    message: str = ""
+
+
+class KaggleState(OEState):
+    task_id: str = ""
+    max_steps: int = 10
+    done: bool = False
+    cv_score: float = 0.0
+    test_score: float = 0.0
+    applied_strategies: List[str] = Field(default_factory=list)
+    strategy_history: List[str] = Field(default_factory=list)
+    leaderboard_rank: int = 0
+    leaderboard: List[Dict[str, Any]] = Field(default_factory=list)
+    submitted: bool = False
+    hint_count: int = 0
+    active_combos: List[str] = Field(default_factory=list)
+    traps_triggered: List[str] = Field(default_factory=list)
+
+
+# ── OpenEnv Environment adapter ──────────────────────────────────────────
+
+class KaggleSimEnvironment(Environment[KaggleAction, KaggleObservation, KaggleState]):
+    """Bridges KaggleSimEnv to the openenv ``Environment`` ABC."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._env = KaggleSimEnv()
+        self._task_id = "easy_churn"
+
+    def reset(
+        self,
+        seed: Optional[int] = None,
+        episode_id: Optional[str] = None,
+        *,
+        task_id: str = "easy_churn",
+        **kwargs: Any,
+    ) -> KaggleObservation:
+        self._task_id = task_id
+        obs = self._env.reset(task_id=task_id)
+        return KaggleObservation(
+            done=obs.done,
+            reward=0.0,
+            dataset_metadata=obs.dataset_metadata.model_dump(),
+            applied_strategies=obs.applied_strategies,
+            current_cv_score=obs.current_cv_score,
+            leaderboard_rank=obs.leaderboard_rank,
+            step_count=obs.step_count,
+            max_steps=obs.max_steps,
+            message=obs.message,
+        )
+
+    def step(
+        self,
+        action: KaggleAction,
+        timeout_s: Optional[float] = None,
+        **kwargs: Any,
+    ) -> KaggleObservation:
+        ks_action = KSAction(
+            action_type=action.action_type,
+            parameters=action.parameters,
+        )
+        result = self._env.step(ks_action)
+        obs = result.observation
+        return KaggleObservation(
+            done=obs.done,
+            reward=result.reward.total,
+            metadata={
+                "info": result.info,
+                "breakdown": result.reward.breakdown.model_dump(),
+            },
+            dataset_metadata=obs.dataset_metadata.model_dump(),
+            applied_strategies=obs.applied_strategies,
+            current_cv_score=obs.current_cv_score,
+            leaderboard_rank=obs.leaderboard_rank,
+            step_count=obs.step_count,
+            max_steps=obs.max_steps,
+            message=obs.message,
+        )
+
+    @property
+    def state(self) -> KaggleState:
+        s = self._env.state()
+        return KaggleState(
+            episode_id=s.task_id,
+            step_count=s.step_count,
+            task_id=s.task_id,
+            max_steps=s.max_steps,
+            done=s.done,
+            cv_score=s.cv_score,
+            test_score=s.test_score,
+            applied_strategies=s.applied_strategies,
+            strategy_history=s.strategy_history,
+            leaderboard_rank=s.leaderboard_rank,
+            leaderboard=s.leaderboard,
+            submitted=s.submitted,
+            hint_count=s.hint_count,
+            active_combos=s.active_combos,
+            traps_triggered=s.traps_triggered,
+        )
+
+    def get_metadata(self) -> EnvironmentMetadata:
+        return EnvironmentMetadata(
+            name="KaggleSimEnv",
+            description=(
+                "RL environment simulating Kaggle competitions with hierarchical "
+                "actions, causal dataset properties, failure-mode traps, and "
+                "contextual scoring."
+            ),
+            version="3.0.0",
+        )
+
+
+# ── Create the OpenEnv app ───────────────────────────────────────────────
+
+app = create_app(
+    KaggleSimEnvironment,
+    KaggleAction,
+    KaggleObservation,
+    env_name="kaggle_sim_env",
+    max_concurrent_envs=1,
+)
+
+
+# ── Custom endpoints (tasks, grader, baseline, actions) ──────────────────
+
+from pydantic import BaseModel
+
+_grader = Grader()
+
+
+class _TaskSummary(BaseModel):
+    task_id: str
+    title: str
+    difficulty: str
+    description: str
+    max_steps: int
+    num_expected_strategies: int
+    num_strategy_combos: int
+    num_failure_modes: int
+
+
+class _BaselineRequest(BaseModel):
+    task_id: str = "easy_churn"
+
+
+class _ActionCategoryEntry(BaseModel):
+    action_type: str
+    parameter_key: Optional[str] = None
+    categories: Dict[str, List[str]] = Field(default_factory=dict)
+
+
+@app.get("/tasks", response_model=List[_TaskSummary], tags=["Custom"])
+def list_tasks() -> list[_TaskSummary]:
+    return [
+        _TaskSummary(
+            task_id=t.task_id,
+            title=t.title,
+            difficulty=t.difficulty,
+            description=t.description,
+            max_steps=t.max_steps,
+            num_expected_strategies=len(t.expected_strategies),
+            num_strategy_combos=len(t.strategy_combos),
+            num_failure_modes=len(t.failure_modes),
+        )
+        for t in TASK_REGISTRY.values()
+    ]
+
+
+@app.post("/grader", response_model=GradeResult, tags=["Custom"])
+def grade(req: _BaselineRequest) -> GradeResult:
+    bl_env = KaggleSimEnv()
+    bl_env.reset(task_id=req.task_id)
+    s = bl_env.state()
+    return _grader.grade(s, get_task(s.task_id))
+
+
+@app.get("/actions", response_model=List[_ActionCategoryEntry], tags=["Custom"])
+def action_space() -> list[_ActionCategoryEntry]:
+    from kaggle_sim_env.models import _PARAM_KEY_MAP
+
+    entries: list[_ActionCategoryEntry] = []
+    for at, key in _PARAM_KEY_MAP.items():
+        cats = get_categories_for_action(at)
+        entries.append(_ActionCategoryEntry(action_type=at, parameter_key=key, categories=cats))
+    entries.append(
+        _ActionCategoryEntry(
+            action_type="pseudo_label",
+            parameter_key="iterations",
+            categories={"iterations": ["1", "2", "3"]},
+        )
+    )
+    entries.append(_ActionCategoryEntry(action_type="inspect_top_solution"))
+    entries.append(_ActionCategoryEntry(action_type="submit"))
+    return entries
+
+
+# ── Entry points ─────────────────────────────────────────────────────────
+
+def main(host: str = "0.0.0.0", port: int = 7860) -> None:
+    import uvicorn
+
+    uvicorn.run(app, host=host, port=port)
+
+
+if __name__ == "__main__":
+    main()

validate.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+ENV_URL=$1
+
+echo "🔍 Checking HF Space..."
+curl -s -X POST "$ENV_URL/reset" \
+  -H "Content-Type: application/json" \
+  -d '{"task_id": "easy_churn"}' > /dev/null
+
+if [ $? -eq 0 ]; then
+  echo "✅ HF Space reachable"
+else
+  echo "❌ HF Space failed"
+  exit 1
+fi
+
+echo "🐳 Checking Docker build..."
+docker build -t test-env . > /dev/null
+
+if [ $? -eq 0 ]; then
+  echo "✅ Docker build passed"
+else
+  echo "❌ Docker build failed"
+  exit 1
+fi
+
+echo "🧠 Checking OpenEnv spec..."
+openenv validate
+
+if [ $? -eq 0 ]; then
+  echo "✅ OpenEnv validation passed"
+else
+  echo "❌ OpenEnv validation failed"
+  exit 1
+fi
+
+echo "🎉 ALL CHECKS PASSED"