commit

inference.py

@@ -1,7 +1,6 @@
 """
 inference.py — Baseline inference script for CodeReview-Env.
-
-Mandatory [START] / [STEP] / [END] log format for OpenEnv evaluators.
+Uses required [START] / [STEP] / [END] plain-text log format.
 
 Environment variables:
   API_BASE_URL   LLM API base URL
@@ -31,38 +30,26 @@ SUCCESS_SCORE_THRESHOLD = 0.6
 TASKS = ["easy_syntax", "medium_logic", "hard_security"]
 
 
-# ── Mandatory log format ──────────────────────────────────────
+# ── MANDATORY log format: plain text [START]/[STEP]/[END] ─────
 
 def log_start(task: str, env: str, model: str) -> None:
-    print(json.dumps({"type": "START", "task": task, "env": env, "model": model}), flush=True)
+    print(f"[START] task={task} env={env} model={model}", flush=True)
 
 
 def log_step(step: int, action: Any, reward: float, done: bool, error: Optional[str] = None) -> None:
-    print(json.dumps({
-        "type": "STEP",
-        "step": step,
-        "action": str(action)[:300],
-        "reward": reward,
-        "done": done,
-        "error": error,
-    }), flush=True)
+    action_str = str(action)[:100].replace("\n", " ")
+    error_str = error if error else "null"
+    print(f"[STEP] step={step} action={action_str} reward={reward} done={done} error={error_str}", flush=True)
 
 
 def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
-    print(json.dumps({
-        "type": "END",
-        "success": success,
-        "steps": steps,
-        "score": score,
-        "rewards": rewards,
-    }), flush=True)
+    rewards_str = str([round(r, 4) for r in rewards])
+    print(f"[END] success={success} steps={steps} score={score} rewards={rewards_str}", flush=True)
 
 
 # ── HTTP client for OpenEnv server ───────────────────────────
 
 class EnvClient:
-    """Thin HTTP client for the OpenEnv-compliant Space API."""
-
     def __init__(self, base_url: str) -> None:
         self.base_url = base_url
         self._http = httpx.Client(timeout=60.0)
@@ -72,7 +59,7 @@ class EnvClient:
             try:
                 r = self._http.get(f"{self.base_url}/health")
                 if r.status_code == 200:
-                    print(f"[DEBUG] Server ready after {i} retries", flush=True)
+                    print(f"[DEBUG] Server ready", flush=True)
                     return True
             except Exception as e:
                 print(f"[DEBUG] Waiting for server ({i+1}/{retries}): {e}", flush=True)
@@ -80,7 +67,6 @@ class EnvClient:
         return False
 
     def reset(self) -> Dict:
-        """POST /reset — returns {observation, reward, done}"""
         try:
             r = self._http.post(f"{self.base_url}/reset")
             r.raise_for_status()
@@ -90,11 +76,9 @@ class EnvClient:
             return {"observation": {}, "reward": 0.0, "done": False}
 
     def step(self, action: Dict) -> Dict:
-        """POST /step with {action: {...}} wrapper — OpenEnv format."""
         try:
-            # OpenEnv create_app requires action wrapped: {"action": {...}}
-            payload = {"action": action}
-            r = self._http.post(f"{self.base_url}/step", json=payload)
+            # OpenEnv create_app requires: {"action": {...}}
+            r = self._http.post(f"{self.base_url}/step", json={"action": action})
             r.raise_for_status()
             return r.json()
         except Exception as e:
@@ -112,9 +96,8 @@ class EnvClient:
 
 SYSTEM_PROMPT = """\
 You are an expert software engineer specialising in code review, debugging, \
-and security auditing. You will be shown a code snippet with a task description.
-
-Return ONLY a JSON object in this exact format (no prose, no markdown fences):
+and security auditing. Analyse the code and return ONLY a JSON object \
+(no prose, no markdown fences):
 
 {
   "identified_issues": [
@@ -126,7 +109,7 @@ Return ONLY a JSON object in this exact format (no prose, no markdown fences):
     }
   ],
   "suggested_fix": "<complete corrected code as string, or null>",
-  "explanation": "<brief summary of findings>",
+  "explanation": "<brief summary of all findings>",
   "submit": true
 }
 """
@@ -173,7 +156,6 @@ def call_llm(llm_client: OpenAI, prompt: str) -> str:
 
 def parse_llm_output(raw: str) -> Dict:
     raw = raw.strip()
-    # Strip markdown fences if present
     if raw.startswith("```"):
         parts = raw.split("```")
         raw = parts[1] if len(parts) > 1 else raw
@@ -185,7 +167,7 @@ def parse_llm_output(raw: str) -> Dict:
         return {
             "identified_issues": [],
             "suggested_fix": None,
-            "explanation": raw[:300],
+            "explanation": raw[:200],
             "submit": True,
         }
 
@@ -204,11 +186,10 @@ def run_task(task_id: str, env: EnvClient, llm: OpenAI) -> float:
     try:
         result = env.reset()
         obs = result.get("observation", {})
-        max_steps = obs.get("max_steps", 5)
+        max_steps = int(obs.get("max_steps", 5))
 
         for step in range(1, max_steps + 1):
-            done = result.get("done", False)
-            if done:
+            if result.get("done", False):
                 break
 
             prompt = build_prompt(obs, step, prev_feedback)
@@ -226,7 +207,7 @@ def run_task(task_id: str, env: EnvClient, llm: OpenAI) -> float:
 
             log_step(
                 step=step,
-                action=action.get("explanation", "")[:200],
+                action=action.get("explanation", "")[:100],
                 reward=reward,
                 done=done,
                 error=None,
@@ -241,6 +222,7 @@ def run_task(task_id: str, env: EnvClient, llm: OpenAI) -> float:
 
     except Exception as e:
         print(f"[DEBUG] run_task error: {e}", flush=True)
+        log_step(step=steps_taken + 1, action="error", reward=0.0, done=True, error=str(e))
         score = 0.0
         success = False
 
@@ -253,34 +235,35 @@ def run_task(task_id: str, env: EnvClient, llm: OpenAI) -> float:
 # ── Main ──────────────────────────────────────────────────────
 
 def main() -> None:
-    print(f"[DEBUG] Starting inference. SPACE_URL={SPACE_URL} MODEL={MODEL_NAME}", flush=True)
+    print(f"[DEBUG] Starting. SPACE_URL={SPACE_URL} MODEL={MODEL_NAME}", flush=True)
 
     llm = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
     env = EnvClient(SPACE_URL)
 
     if not env.wait_until_ready():
         print("[ERROR] Server not reachable. Exiting.", flush=True)
+        for task_id in TASKS:
+            log_start(task=task_id, env=BENCHMARK, model=MODEL_NAME)
+            log_end(success=False, steps=0, score=0.0, rewards=[])
         sys.exit(1)
 
     task_scores: Dict[str, float] = {}
 
     for task_id in TASKS:
-        print(f"\n{'='*50}\nRunning: {task_id}\n{'='*50}", flush=True)
+        print(f"\n[DEBUG] ===== Running task: {task_id} =====", flush=True)
         try:
             task_scores[task_id] = run_task(task_id, env, llm)
         except Exception as e:
             print(f"[DEBUG] Task {task_id} crashed: {e}", flush=True)
+            log_start(task=task_id, env=BENCHMARK, model=MODEL_NAME)
+            log_end(success=False, steps=0, score=0.0, rewards=[])
             task_scores[task_id] = 0.0
         time.sleep(1)
 
     env.close()
 
-    print(f"\n{'='*50}\nFINAL SCORES\n{'='*50}", flush=True)
-    for tid, s in task_scores.items():
-        status = "PASS" if s >= SUCCESS_SCORE_THRESHOLD else "FAIL"
-        print(f"  {tid:25s}: {s:.4f}  [{status}]", flush=True)
     overall = sum(task_scores.values()) / len(task_scores)
-    print(f"\n  Overall: {overall:.4f}", flush=True)
+    print(f"\n[DEBUG] Overall average: {overall:.4f}", flush=True)
 
 
 if __name__ == "__main__":