Upload folder using huggingface_hub

inference.py

@@ -2,29 +2,21 @@
 """
 AgentOps Gym — Baseline inference script.
 
-Uses the synchronous OpenEnv client pattern (env.sync()) matching the
-hackathon sample inference.py. No async/await needed.
-
-Environment variables:
-    IMAGE_NAME     Docker image name (set by validator)
-    HF_TOKEN       HuggingFace / API key  (or OPENAI_API_KEY)
-    API_BASE_URL   LLM endpoint  (default: https://router.huggingface.co/v1)
-    MODEL_NAME     Model name    (default: Qwen/Qwen2.5-72B-Instruct)
-    ENV_BASE_URL   Server URL    (default: http://localhost:8000)
-
-Usage:
-    IMAGE_NAME=agentops-gym HF_TOKEN=xxx python inference.py
-"""
+Follows the exact pattern from the official OpenEnv sample inference.py.
 
-from __future__ import annotations
+STDOUT FORMAT:
+    [START] task=<task> env=<benchmark> model=<model>
+    [STEP]  step=<n> action=<str> reward=<0.00> done=<true|false> error=<msg|null>
+    [END]   success=<true|false> steps=<n> score=<0.000> rewards=<r1,r2,...>
+"""
 
+import asyncio
 import json
 import os
 import re
-import sys
 from typing import Dict, List, Optional
 
-# Load .env if present
+# Load .env if present (local dev only)
 try:
     from dotenv import load_dotenv
     load_dotenv()
@@ -33,7 +25,7 @@ except ImportError:
 
 from openai import OpenAI
 
-# Ensure package is importable when run from inside the package dir
+# Ensure package importable from any working directory
 import pathlib, sys as _sys
 _root = pathlib.Path(__file__).resolve().parent
 _parent = _root.parent
@@ -45,18 +37,14 @@ from agentops_gym.client import AgentOpsEnv
 from agentops_gym.models import ToolCall
 
 # ---------------------------------------------------------------------------
-# Configuration
+# Configuration — exactly matching the official sample pattern
 # ---------------------------------------------------------------------------
 
-IMAGE_NAME   = os.getenv("IMAGE_NAME")
-API_KEY      = (
-    os.getenv("HF_TOKEN")
-    or os.getenv("OPENAI_API_KEY")
-    or os.getenv("API_KEY")
-)
-API_BASE_URL = os.getenv("API_BASE_URL", "https://router.huggingface.co/v1")
-MODEL_NAME   = os.getenv("MODEL_NAME", "Qwen/Qwen2.5-72B-Instruct")
-ENV_BASE_URL = os.getenv("ENV_BASE_URL", "http://localhost:8000")
+IMAGE_NAME = os.getenv("IMAGE_NAME")
+API_KEY    = os.getenv("HF_TOKEN") or os.getenv("API_KEY")  # HF_TOKEN first
+
+API_BASE_URL = os.getenv("API_BASE_URL") or "https://router.huggingface.co/v1"
+MODEL_NAME   = os.getenv("MODEL_NAME")   or "Qwen/Qwen2.5-72B-Instruct"
 
 BENCHMARK               = "agentops-gym"
 MAX_STEPS               = 10
@@ -101,16 +89,18 @@ def log_start(task: str, env: str, model: str) -> None:
 
 
 def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]) -> None:
-    err_val = error if error else "null"
+    error_val = error if error else "null"
+    done_val  = str(done).lower()
     print(
-        f"[STEP] step={step} action={str(action).replace(chr(10), ' ')[:200]} "
-        f"reward={reward:.2f} done={str(done).lower()} error={err_val}",
+        f"[STEP] step={step} action={str(action).replace(chr(10),' ')[:200]} "
+        f"reward={reward:.2f} done={done_val} error={error_val}",
         flush=True,
     )
 
 
 def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
-    score = max(0.001, min(0.999, score))  # must be strictly between 0 and 1
+    # Score must be strictly between 0 and 1
+    score = max(0.001, min(0.999, score))
     rewards_str = ",".join(f"{r:.2f}" for r in rewards)
     print(
         f"[END] success={str(success).lower()} steps={steps} "
@@ -166,11 +156,10 @@ def extract_tool_call(text: str) -> Optional[Dict]:
     return None
 
 
-def get_model_action(client: OpenAI, obs_data: Dict, history: List[str], model_name: str = MODEL_NAME) -> Optional[Dict]:
-    """Ask the LLM for a tool call. Returns parsed dict or None."""
+def get_model_action(client: OpenAI, obs_data: Dict, history: List[str]) -> Optional[Dict]:
     try:
         completion = client.chat.completions.create(
-            model=model_name,
+            model=MODEL_NAME,
             messages=[
                 {"role": "system", "content": SYSTEM_PROMPT},
                 {"role": "user",   "content": build_prompt(obs_data, history)},
@@ -185,34 +174,49 @@ def get_model_action(client: OpenAI, obs_data: Dict, history: List[str], model_n
         return None
 
 # ---------------------------------------------------------------------------
-# Single task runner — sync pattern matching sample inference.py
+# Single episode runner
 # ---------------------------------------------------------------------------
 
-def run_task(client: OpenAI, task_id: str, model_name: str = MODEL_NAME) -> Dict:
-    """Run one episode synchronously. Returns result dict."""
-
-    # Build client — use docker image if set, else connect to running server
-    if IMAGE_NAME:
-        env_client = AgentOpsEnv.from_docker_image(IMAGE_NAME)
-    else:
-        env_client = AgentOpsEnv(base_url=ENV_BASE_URL)
-
-    history: List[str] = []
-    rewards: List[float] = []
+async def run_episode(env: AgentOpsEnv, client: OpenAI, task_id: str) -> Dict:
+    history:  List[str]   = []
+    rewards:  List[float] = []
     steps_taken = 0
-    score = 0.0
-    success = False
-    last_error: Optional[str] = None
+    score       = 0.001
+    success     = False
+    obs_data: Dict = {}
 
     log_start(task=task_id, env=BENCHMARK, model=MODEL_NAME)
 
     try:
-        # Use .sync() context manager — same pattern as sample inference.py
-        with env_client.sync() as env:
-            if IMAGE_NAME:
-                result = env.reset()
-            else:
-                result = env.reset(task_id=task_id)
+        result   = await env.reset(seed=None, task_id=task_id)
+        obs_data = (
+            result.observation.model_dump()
+            if hasattr(result.observation, "model_dump")
+            else result.observation.dict()
+        )
+
+        for step in range(1, MAX_STEPS + 1):
+            if result.done or obs_data.get("done", False):
+                break
+
+            tool_call = get_model_action(client, obs_data, history) or {
+                "tool": "Grep",
+                "parameters": {"pattern": "def "},
+                "reasoning": "fallback",
+            }
+
+            tool      = tool_call.get("tool", "Grep")
+            params    = tool_call.get("parameters", {})
+            reasoning = tool_call.get("reasoning", "")
+            action_str = f"{tool}({json.dumps(params)})"
+
+            try:
+                result = await env.step(
+                    ToolCall(tool=tool, parameters=params, reasoning=reasoning)
+                )
+            except Exception as e:
+                log_step(step=step, action=action_str, reward=0.0, done=True, error=str(e))
+                break
 
             obs_data = (
                 result.observation.model_dump()
@@ -220,63 +224,28 @@ def run_task(client: OpenAI, task_id: str, model_name: str = MODEL_NAME) -> Dict
                 else result.observation.dict()
             )
 
-            for step in range(1, MAX_STEPS + 1):
-                if result.done or obs_data.get("done", False):
-                    break
-
-                tool_call = get_model_action(client, obs_data, history, model_name)
-                if tool_call is None:
-                    tool_call = {
-                        "tool": "Grep",
-                        "parameters": {"pattern": "def "},
-                        "reasoning": "fallback",
-                    }
-
-                tool      = tool_call.get("tool", "Grep")
-                params    = tool_call.get("parameters", {})
-                reasoning = tool_call.get("reasoning", "")
-                action_str = f"{tool}({json.dumps(params)})"
-
-                try:
-                    result = env.step(
-                        ToolCall(tool=tool, parameters=params, reasoning=reasoning)
-                    )
-                    last_error = None
-                except Exception as e:
-                    last_error = str(e)
-                    log_step(step=step, action=action_str, reward=0.0, done=True, error=last_error)
-                    break
-
-                obs_data = (
-                    result.observation.model_dump()
-                    if hasattr(result.observation, "model_dump")
-                    else result.observation.dict()
-                )
-
-                reward = float(result.reward or 0.0)
-                done   = bool(result.done)
+            reward = float(result.reward or 0.0)
+            done   = bool(result.done)
 
-                rewards.append(reward)
-                steps_taken = step
-                history.append(f"Step {step}: {action_str} → reward {reward:.2f}")
+            rewards.append(reward)
+            steps_taken = step
+            history.append(f"Step {step}: {action_str} → reward {reward:.2f}")
 
-                log_step(step=step, action=action_str, reward=reward, done=done, error=None)
+            log_step(step=step, action=action_str, reward=reward, done=done, error=None)
 
-                if done:
-                    break
+            if done:
+                break
 
-        # Pull grader score from last observation metadata
         meta  = obs_data.get("metadata", {})
         score = float(meta.get("grader_score") or 0.0)
-        if score == 0.0 and rewards:
+        if score == 0.0:
             score = float(meta.get("cumulative_reward") or 0.0)
         score   = max(0.001, min(0.999, score))
         success = score >= SUCCESS_SCORE_THRESHOLD
 
     except Exception as e:
-        print(f"[DEBUG] Task {task_id} error: {e}", flush=True)
-        last_error = str(e)
-        score = 0.001  # never exactly 0.0
+        print(f"[DEBUG] Episode error for {task_id}: {e}", flush=True)
+        score = 0.001
 
     finally:
         log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
@@ -290,44 +259,32 @@ def run_task(client: OpenAI, task_id: str, model_name: str = MODEL_NAME) -> Dict
     }
 
 # ---------------------------------------------------------------------------
-# Entry point
+# Main — exactly matching official sample pattern
 # ---------------------------------------------------------------------------
 
-def main() -> None:
-    # Validator requires EXACTLY these — square brackets, no fallback
-    api_key      = os.environ["API_KEY"]
-    api_base_url = os.environ["API_BASE_URL"]
-    model_name   = os.environ.get("MODEL_NAME", "Qwen/Qwen2.5-72B-Instruct")
-
-    print(f"[DEBUG] API_BASE_URL={api_base_url}", flush=True)
-    print(f"[DEBUG] MODEL_NAME={model_name}", flush=True)
+async def async_main() -> None:
+    # Use module-level API_KEY and API_BASE_URL — same as official sample
+    client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
 
-    client = OpenAI(base_url=api_base_url, api_key=api_key)
+    # from_docker_image is awaitable — same as official sample
+    env = await AgentOpsEnv.from_docker_image(IMAGE_NAME)
 
     print("=" * 60, flush=True)
     print("AgentOps Gym — Baseline Inference", flush=True)
-    print(f"Model: {model_name}  |  Image: {IMAGE_NAME or ENV_BASE_URL}", flush=True)
+    print(f"Model: {MODEL_NAME}  |  Image: {IMAGE_NAME}", flush=True)
     print("=" * 60, flush=True)
 
     results = []
-    for task_id in ALL_TASKS:
-        print("─" * 40, flush=True)
-        results.append(run_task(client, task_id, model_name))
-
-    total  = sum(r["score"] for r in results)
-    solved = sum(1 for r in results if r["success"])
-    avg    = total / len(results) if results else 0.0
-
-    print("=" * 60, flush=True)
-    print("BASELINE SUMMARY", flush=True)
-    print("=" * 60, flush=True)
-    for r in results:
-        status = "✅ PASS" if r["success"] else "❌ FAIL"
-        print(f"  {r['task_id']:>8}    score={r['score']:.3f}  steps={r['steps']:2d}  {status}", flush=True)
-    print(f"\n  Average score: {avg:.3f}", flush=True)
-    print(f"  Solved: {solved} / {len(results)}", flush=True)
-    print("=" * 60, flush=True)
+    try:
+        async with env:
+            for task_id in ALL_TASKS:
+                print("─" * 40, flush=True)
+                result = await run_episode(env, client, task_id)
+                results.append(result)
+    except Exception as e:
+        print(f"[DEBUG] Cleanup error (non-fatal): {e}", flush=True)
 
+    # Summary
     total  = sum(r["score"] for r in results)
     solved = sum(1 for r in results if r["success"])
     avg    = total / len(results) if results else 0.0
@@ -344,4 +301,4 @@ def main() -> None:
 
 
 if __name__ == "__main__":
-    main()
+    asyncio.run(async_main())