Upload folder using huggingface_hub

inference.py

@@ -1,15 +1,52 @@
+"""
+inference.py — Rust Coder OpenEnv Baseline Agent
+
+Architecture
+────────────
+• Runs 3 tasks (easy / medium / hard) as independent episodes.
+• Each task produces its own [START]…[STEP]…[END] log block.
+• A fresh WebSocket env connection is opened per task to avoid
+  HF-Space WebSocket timeouts during long LLM + compilation waits.
+• Scores are clamped to (0.01, 0.99) — strictly inside (0, 1).
+• If HF_TOKEN is missing, minimal fallback blocks are emitted so
+  the platform always receives 3 parseable task records.
+
+Required env vars
+─────────────────
+    API_BASE_URL  — LLM router URL   (default: HF router)
+    MODEL_NAME    — model identifier (default: Qwen 72B)
+    HF_TOKEN      — HuggingFace / API key
+    ENV_URL       — environment URL  (default: http://localhost:8000)
+"""
+
 import os
-import re
-import json
 import asyncio
 import logging
 from typing import List, Optional
+
 from openai import OpenAI
 from dotenv import load_dotenv
 
 load_dotenv()
 
-# --- Logging (inference.py) ---
+# ── Configuration ─────────────────────────────────────────────────────────────
+API_BASE_URL = os.getenv("API_BASE_URL") or "https://router.huggingface.co/v1"
+MODEL_NAME   = os.getenv("MODEL_NAME")   or "Qwen/Qwen2.5-72B-Instruct"
+HF_TOKEN     = os.getenv("HF_TOKEN")    or os.getenv("API_KEY")
+ENV_URL      = os.getenv("ENV_URL")     or "http://localhost:8000"
+
+SUCCESS_SCORE_THRESHOLD = 0.5
+TEMPERATURE  = 0.1
+MAX_TOKENS   = 1500
+
+# Exactly 3 tasks: easy / medium / hard (maps to problems.json indices)
+EVAL_TASKS = [
+    {"task_id": "task_1", "start_index": 0, "difficulty": "easy"},
+    {"task_id": "task_3", "start_index": 2, "difficulty": "medium"},
+    {"task_id": "task_6", "start_index": 5, "difficulty": "hard"},
+]
+
+# ── Logging ───────────────────────────────────────────────────────────────────
 _LOG_LEVEL = (os.getenv("LOG_LEVEL") or "INFO").upper()
 logging.basicConfig(
     level=getattr(logging, _LOG_LEVEL, logging.INFO),
@@ -17,146 +54,178 @@ logging.basicConfig(
 )
 logger = logging.getLogger("rust_coder.inference")
 
-# --- Competition Configuration ---
-API_BASE_URL = os.getenv("API_BASE_URL") or "https://router.huggingface.co/v1"
-MODEL_NAME = os.getenv("MODEL_NAME") or "Qwen/Qwen2.5-72B-Instruct"
-HF_TOKEN = os.getenv("HF_TOKEN") or os.getenv("API_KEY")
-ENV_URL = os.getenv("ENV_URL") or "http://localhost:8000"
-
-# Episode constants: 10 problems, each worth max reward 1.0
-MAX_STEPS = 10
-MAX_TOTAL_REWARD = 10.0
-SUCCESS_SCORE_THRESHOLD = 0.5
-
-# Import client (ensure rust_coder is in PYTHONPATH)
 from client import RustCoderEnv
 from models import RustCoderAction
 
-# --- Strict Logging Helpers ---
-def log_start(task: str, env: str, model: str):
-    # REQUIRED exact stdout format (no quotes)
+
+# ── Strict stdout log helpers ─────────────────────────────────────────────────
+
+def log_start(task: str, env: str, model: str) -> None:
     print(f"[START] task={task} env={env} model={model}", flush=True)
 
-def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str] = None):
-    # REQUIRED exact stdout format:
-    # [STEP] step=<n> action=<action_str> reward=<0.00> done=<true|false> error=<msg|null>
-    action_str = (action or "").replace("\r", "\\r").replace("\n", "\\n")
-    action_str = action_str[:200]  # keep single-line + bounded
-    err_field = "null" if error is None else str(error).replace("\r", "\\r").replace("\n", "\\n")
-    reward_2 = f"{float(reward or 0.0):.2f}"
+
+def log_step(
+    step: int,
+    action: str,
+    reward: float,
+    done: bool,
+    error: Optional[str] = None,
+) -> None:
+    action_str = (action or "").replace("\r", "\\r").replace("\n", "\\n")[:200]
+    err_field  = "null" if error is None else str(error).replace("\n", "\\n")[:200]
     print(
-        f"[STEP] step={step} action={action_str} reward={reward_2} done={str(bool(done)).lower()} error={err_field}",
+        f"[STEP] step={step} action={action_str} reward={reward:.2f} "
+        f"done={str(bool(done)).lower()} error={err_field}",
         flush=True,
     )
 
-def log_end(success: bool, steps: int, score: float, rewards: List[float]):
-    # REQUIRED exact stdout format, rewards as comma-separated 2dp
-    rewards_str = ",".join(f"{float(r or 0.0):.2f}" for r in rewards)
+
+def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
+    rewards_str = ",".join(f"{r:.2f}" for r in rewards)
     print(
-        f"[END] success={str(bool(success)).lower()} steps={steps} score={float(score or 0.0):.3f} rewards={rewards_str}",
+        f"[END] success={str(success).lower()} steps={steps} "
+        f"score={score:.3f} rewards={rewards_str}",
         flush=True,
     )
 
-# --- LLM Solution Logic ---
+
+# ── Score clamping ────────────────────────────────────────────────────────────
+
+def clamp_score(raw: float) -> float:
+    """
+    Clamp to the open interval (0, 1) — never exactly 0.0 or 1.0.
+
+    Floor 0.01: even compilation failures yield a non-zero score.
+    Ceiling 0.99: prevents a theoretically-perfect submission from
+    returning 1.0.
+    """
+    return round(max(0.01, min(0.99, float(raw))), 3)
+
+
+# ── LLM call ─────────────────────────────────────────────────────────────────
+
 async def get_model_code(prompt: str, client: OpenAI) -> str:
-    """Call the LLM to get a Rust solution."""
+    """Ask the model for a complete Rust solution; strip markdown if needed."""
     try:
-        logger.info(
-            "LLM call start model=%s base_url=%s prompt_chars=%d token_present=%s",
-            MODEL_NAME,
-            API_BASE_URL,
-            len(prompt or ""),
-            bool(HF_TOKEN),
-        )
         completion = client.chat.completions.create(
             model=MODEL_NAME,
             messages=[
-                {"role": "system", "content": "You are a senior Rust systems engineer. Return ONLY the complete, fixed Rust code. No explanation."},
+                {
+                    "role": "system",
+                    "content": (
+                        "You are a senior Rust systems engineer. "
+                        "Return ONLY the complete, corrected Rust source file. "
+                        "No markdown fences. No commentary."
+                    ),
+                },
                 {"role": "user", "content": prompt},
             ],
-            temperature=0.1,
+            temperature=TEMPERATURE,
+            max_tokens=MAX_TOKENS,
         )
         text = (completion.choices[0].message.content or "").strip()
-        logger.debug("LLM raw response chars=%d", len(text))
-
-        # Extract code from markdown
         if "```rust" in text:
             text = text.split("```rust")[1].split("```")[0]
         elif "```" in text:
             text = text.split("```")[1].split("```")[0]
         text = text.strip()
-        if not text:
-            logger.warning("LLM returned empty code after cleanup.")
-            return "// Error: empty response (no code returned)."
-        logger.info("LLM call end: returned_code_chars=%d", len(text))
-        return text
-    except Exception as e:
-        logger.exception("LLM Request failed.")
-        return f"// Error: {e}"
-
-# --- Main Evaluation Loop ---
-async def main():
-    if not HF_TOKEN:
-        logger.error("HF_TOKEN/API_KEY not found in environment.")
-        return
-
-    client = OpenAI(base_url=API_BASE_URL, api_key=HF_TOKEN)
-    env = RustCoderEnv(base_url=ENV_URL)
+        return text or "// empty response"
+    except Exception as exc:
+        logger.exception("LLM call failed")
+        return f"// LLM error: {exc}"
+
+
+# ── Single-task episode ───────────────────────────────────────────────────────
+
+async def run_task(task_info: dict, client: Optional[OpenAI]) -> None:
+    """
+    Run one task as a fully independent episode with its own env connection.
+
+    Opens a fresh WebSocket connection so a slow LLM call on a previous
+    task cannot cause a connection timeout here.
 
-    log_start(task="rust_coder", env="RustCoder-v1", model=MODEL_NAME)
+    Always emits exactly one [START]…[STEP]…[END] block.
+    """
+    task_id     = task_info["task_id"]
+    start_index = task_info["start_index"]
+
+    log_start(task=task_id, env="RustCoder-v1", model=MODEL_NAME)
 
     rewards: List[float] = []
     steps_taken = 0
-    score = 0.0
+    score  = 0.01
     success = False
 
+    # Fresh connection per task — avoids WebSocket timeout across tasks
+    env = RustCoderEnv(base_url=ENV_URL)
     try:
-        # Start the single episode (10 problems)
-        result = await env.reset()
-        obs = result.observation
-
-        for step in range(1, MAX_STEPS + 1):
-            if result.done:
-                break
-
-            steps_taken = step
-
-            # Format prompt including header_section if available
-            prompt = obs.problem_description
-            if getattr(obs, "header_section", ""):
-                prompt += f"\n\nHeader Section (must be included verbatim in final code):\n```rust\n{obs.header_section}\n```"
+        # ── Reset to the target task ──────────────────────────────────
+        reset_result = await env.reset(start_index=start_index)
+        obs = reset_result.observation
+
+        # ── Build prompt ──────────────────────────────────────────────
+        prompt = obs.problem_description or ""
+        header = getattr(obs, "header_section", "")
+        if header:
+            prompt += (
+                "\n\nHeader section (must be included verbatim):"
+                f"\n```rust\n{header}\n```"
+            )
+
+        # ── Get LLM code or skip if no token ─────────────────────────
+        if client is not None:
+            code = await get_model_code(prompt, client)
+        else:
+            code = "// no HF_TOKEN — using stub"
+
+        steps_taken = 1
+
+        # ── Evaluate in environment ───────────────────────────────────
+        step_result = await env.step(RustCoderAction(code=code))
+        # Explicit None check — 0.0 is falsy but valid
+        raw_reward = float(step_result.reward if step_result.reward is not None else 0.0)
+        score  = clamp_score(raw_reward)
+        rewards.append(score)
+        success = score >= SUCCESS_SCORE_THRESHOLD
 
-            # 1. Ask model for solution to current task
-            code_solution = await get_model_code(prompt, client)
+        log_step(step=1, action=code, reward=score, done=True, error=None)
+
+    except Exception as exc:
+        logger.exception("Task %s failed", task_id)
+        score   = 0.01
+        rewards = [0.01]
+        log_step(
+            step=steps_taken + 1,
+            action="error",
+            reward=0.01,
+            done=True,
+            error=str(exc),
+        )
+    finally:
+        try:
+            await env.close()
+        except Exception:
+            pass
 
-            # 2. Environment step
-            logger.debug("Submitting to env.step code_chars=%d", len(code_solution or ""))
-            result = await env.step(RustCoderAction(code=code_solution))
-            obs = result.observation
-            reward = result.reward or 0.0
-            done = result.done
+    log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
 
-            rewards.append(reward)
-            log_step(step=step, action=code_solution, reward=reward, done=done, error=None)
 
-            if done:
-                break
+# ── Main ──────────────────────────────────────────────────────────────────────
 
-        # Normalize score to [0, 1] matching sample format
-        score = sum(rewards) / MAX_TOTAL_REWARD if MAX_TOTAL_REWARD > 0 else 0.0
-        score = min(max(score, 0.0), 1.0)
-        success = score >= SUCCESS_SCORE_THRESHOLD
+async def main() -> None:
+    # Build the LLM client if credentials are available
+    client: Optional[OpenAI] = None
+    if HF_TOKEN:
+        client = OpenAI(base_url=API_BASE_URL, api_key=HF_TOKEN)
+    else:
+        logger.warning(
+            "HF_TOKEN / API_KEY not set — LLM calls disabled. "
+            "Stub code will be submitted; scores will be at floor (0.01)."
+        )
 
-    except Exception as e:
-        logger.exception("Runtime error.")
-        log_step(step=steps_taken + 1, action="error", reward=0.0, done=True, error=str(e))
+    for task in EVAL_TASKS:
+        await run_task(task, client)
 
-    finally:
-        try:
-            await env.close()
-        except Exception as e:
-            logger.exception("env.close() error.")
-        log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
 
 if __name__ == "__main__":
     asyncio.run(main())

server/app.py

@@ -150,10 +150,10 @@ async def grader(task_id: str, action: RustCoderAction):
     Body:   {"code": "<rust source code>"}
 
     Scores are strictly in the open interval (0, 1):
-      - Minimum 0.01  — floor for any submission (even empty/non-compiling code)
-      - Maximum 0.99  — ceiling so no submission scores a theoretical perfect 1.0
-      - Natural range based on: Compilation(40%) + Correctness(20%) +
-        Coverage(20%) + Elegance(10%) + Efficiency(10%)
+      - Minimum 0.01  — floor for any submission (even empty/non-compiling)
+      - Maximum 0.99  — ceiling so no submission hits the theoretical perfect
+      - Weighted: Compilation(40%) + Correctness(20%) + Coverage(20%) +
+                  Elegance(10%) + Efficiency(10%)
     """
     task_meta = TASK_REGISTRY.get(task_id)
     if task_meta is None:
@@ -162,25 +162,49 @@ async def grader(task_id: str, action: RustCoderAction):
             detail=f"Unknown task_id '{task_id}'. Valid IDs: {TASK_IDS}",
         )
 
+    _EMPTY_BREAKDOWN = {
+        "compilation": 0.0,
+        "correctness": 0.0,
+        "coverage":    0.0,
+        "elegance":    0.0,
+        "efficiency":  0.0,
+    }
+
+    # Fast path: empty code — skip compilation + avoid triggering auto-LLM
+    if not action.code.strip():
+        return {
+            "task_id":            task_id,
+            "score":              0.01,
+            "passed":             0,
+            "total":              1,
+            "metric":             "rust_code_quality",
+            "reward_breakdown":   _EMPTY_BREAKDOWN,
+            "compilation_success": False,
+            "compilation_output": "No code submitted.",
+            "test_results":       [],
+        }
+
+    # Full evaluation path
     env = RustCoderEnvironment()
     env.reset(start_index=task_meta["index"])
     obs = env.step(action)
 
-    raw_score = float(obs.reward) if obs.reward is not None else 0.0
+    # Explicit None check — 0.0 is falsy but a valid reward
+    raw_score = float(obs.reward if obs.reward is not None else 0.0)
     # Enforce strictly open interval (0, 1) — never exactly 0.0 or 1.0
-    score = round(max(0.01, min(0.99, raw_score)), 4)
+    score   = round(max(0.01, min(0.99, raw_score)), 4)
     success = score >= task_meta["success_threshold"]
 
     return {
-        "task_id": task_id,
-        "score": score,
-        "passed": 1 if success else 0,
-        "total": 1,
-        "metric": "rust_code_quality",
-        "reward_breakdown": obs.reward_breakdown,
+        "task_id":             task_id,
+        "score":               score,
+        "passed":              1 if success else 0,
+        "total":               1,
+        "metric":              "rust_code_quality",
+        "reward_breakdown":    obs.reward_breakdown,
         "compilation_success": obs.compilation_success,
-        "compilation_output": obs.compilation_output,
-        "test_results": obs.test_results,
+        "compilation_output":  obs.compilation_output,
+        "test_results":        obs.test_results,
     }