Fix final phase 1 and phase 2 inference and scoring issues

client.py

@@ -1,4 +1,10 @@
-from openenv.core.client import EnvClient
+try:
+    from openenv.core.client import EnvClient
+except ImportError:
+    try:
+        from openenv.core.env_client import EnvClient
+    except ImportError:
+        from openenv.core import EnvClient
 from models import APIAction, APIObservation, APIState
 
 

graders/grader.py

@@ -34,17 +34,8 @@ class APIGrader:
     def _schema_match(self, body, schema):
         if not schema:
             return 1.0
-            
-        schemas = schema if isinstance(schema, list) else [schema]
-        best_match = 0.0
-        
-        for s in schemas:
-            matched = sum(1 for k, v in s.items() if k in body and type(body[k]) == type(v))
-            score = matched / len(s) if len(s) > 0 else 1.0
-            if score > best_match:
-                best_match = score
-                
-        return best_match
+        matched = sum(1 for k, v in schema.items() if k in body and type(body[k]) == type(v))
+        return matched / len(schema) if len(schema) > 0 else 1.0
 
     def get_feedback(self, status, expected_status):
         if status == expected_status:

inference.py

@@ -47,7 +47,7 @@ Respond ONLY in valid JSON with exactly these fields:
 }
 Rules:
 - method must be uppercase (GET, POST, DELETE, etc.)
-- url must start with /mock_api/
+- url must start with / (e.g. /mock_api/... for standard endpoints, or /openapi.json for discovery tasks)
 - body must be null for GET/DELETE requests
 - always include Content-Type: application/json when body is not null
 - do not add any text outside the JSON
@@ -64,7 +64,7 @@ def log_step(step: int, action: str, reward: float, done: bool,
     error_val = error if error else "null"
     done_val  = str(done).lower()
     print(
-        f"[STEP] step={step} action={action} reward={reward:.2f}"
+        f"[STEP] step={step} action={action} reward={reward:.3f}"
         f" done={done_val} error={error_val}",
         flush=True,
     )
@@ -72,7 +72,7 @@ def log_step(step: int, action: str, reward: float, done: bool,
 
 def log_end(success: bool, steps: int, score: float,
             rewards: List[float]) -> None:
-    rewards_str = ",".join(f"{r:.2f}" for r in rewards)
+    rewards_str = ",".join(f"{r:.3f}" for r in rewards)
     print(
         f"[END] success={str(success).lower()} steps={steps}"
         f" score={score:.3f} rewards={rewards_str}",
@@ -222,12 +222,12 @@ async def run_task(client: httpx.AsyncClient, task_id: str) -> float:
             # ── Step ──────────────────────────────────────────────────────────
             try:
                 result = await env_step(client, action)
-                reward = float(result.get("reward", 0.0))
+                reward = float(result.get("reward", 0.001))
                 done   = bool(result.get("done", False))
                 obs    = result.get("observation", result)
                 error_str: Optional[str] = None
             except RuntimeError as exc:
-                reward    = 0.0
+                reward    = 0.001
                 done      = True          # abort episode on env error
                 error_str = str(exc)[:120]
 
@@ -240,7 +240,7 @@ async def run_task(client: httpx.AsyncClient, task_id: str) -> float:
             history.append(
                 f"step={step} action={action_str} "
                 f"status={obs.get('last_status_code', '?')} "
-                f"reward={reward:.2f}"
+                f"reward={reward:.3f}"
             )
 
             if done:

issues_and_fixes.txt

@@ -0,0 +1,32 @@
+API DEBUG ENV — REMAINING ISSUES & FIXES
+==========================================
+
+1. server/Dockerfile port still 8000
+   Fix: Change EXPOSE 8000 → EXPOSE 7860, healthcheck URL to localhost:7860, and CMD port to 7860.
+
+2. MOCK_BASE hardcoded in api_debug_environment.py
+   Fix: Replace `MOCK_BASE = "http://localhost:7860"` with `MOCK_BASE = os.getenv("MOCK_BASE_URL", "http://localhost:7860")` and add `import os` at top.
+
+3. resp_body not capped in api_debug_environment.py
+   Fix: Add `MAX_RESPONSE_BODY_LENGTH = 2000` at top and change `resp_body = resp.text` to `resp_body = resp.text[:MAX_RESPONSE_BODY_LENGTH]`.
+
+4. APIState.visited_endpoints type will break Pydantic v2
+   Fix: In models.py add `from typing import Set` and change `visited_endpoints: set` to `visited_endpoints: Set[str]`.
+
+5. expert_stateful_chain body has literal placeholder key "{dynamic_item_field}"
+   Fix: In tasks/expert.py change the broken_request body from `{"{dynamic_item_field}": "temporary_item"}` to `{"name": "temporary_item"}`. The dynamic field feature doesn't format body keys, only description strings.
+
+6. SYSTEM_PROMPT in inference.py blocks GET /openapi.json for expert discovery task
+   Fix: Change the rule `url must start with /mock_api/` to `url must start with / (e.g. /mock_api/... or /openapi.json for discovery tasks)`.
+
+7. No tests in submission
+   Fix: Add a tests/ directory with at minimum test_environment.py covering grader unit tests, task registry validation, and a mocked episode lifecycle. Run with pytest tests/ -v.
+
+8. _schema_match list-handling code in grader.py is dead/unreachable
+   Fix: Remove the `schemas = schema if isinstance(schema, list) else [schema]` block — the environment already passes a single dict (target_schema) at the call site. The dead code adds confusion without benefit.
+
+9. inference.py SYSTEM_PROMPT temperature note — minor
+   Fix: Already set to 0.0 (deterministic), good. No change needed.
+
+10. hard_token_exchange description uses {client_id}/{client_secret} placeholders
+    These are formatted correctly in reset() so this works. No fix needed — just confirming it's intentional.

models.py

@@ -8,6 +8,7 @@ except ImportError:
     except ImportError:
         from openenv.core.env_server import Action, Observation, State
 
+from typing import Set
 from pydantic import Field
 
 
@@ -39,5 +40,5 @@ class APIState(State):
     max_steps: int = Field(default=10)
     solved: bool = Field(default=False)
     current_step_index: int = Field(default=0)
-    visited_endpoints: set = Field(default_factory=set)
+    visited_endpoints: Set[str] = Field(default_factory=set)
     curriculum_level: int = Field(default=0)

server/Dockerfile

@@ -14,9 +14,9 @@ RUN pip install --no-cache-dir -e .
 ENV PYTHONPATH=/app
 
 USER envuser
-EXPOSE 8000
+EXPOSE 7860
 
 HEALTHCHECK --interval=20s --timeout=8s --start-period=40s --retries=3 \
-    CMD curl -f http://localhost:8000/health || exit 1
+    CMD curl -f http://localhost:7860/health || exit 1
 
-CMD ["uvicorn", "server.app:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "1"]
+CMD ["uvicorn", "server.app:app", "--host", "0.0.0.0", "--port", "7860", "--workers", "1"]

server/api_debug_environment.py

@@ -1,3 +1,4 @@
+import os
 import random
 import httpx
 
@@ -11,7 +12,8 @@ from tasks.registry import TASK_REGISTRY
 from graders.grader import APIGrader
 
 GRADER = APIGrader()
-MOCK_BASE = "http://localhost:7860"
+MOCK_BASE = os.getenv("MOCK_BASE_URL", "http://localhost:7860")
+MAX_RESPONSE_BODY_LENGTH = 2000
 
 
 class APIDebugEnvironment(Environment):
@@ -104,7 +106,7 @@ class APIDebugEnvironment(Environment):
                 )
             status = resp.status_code
             resp_headers = dict(resp.headers)
-            resp_body = resp.text
+            resp_body = resp.text[:MAX_RESPONSE_BODY_LENGTH]
         except Exception as e:
             status = 0
             resp_headers = {}

tasks/easy.py

@@ -11,6 +11,7 @@ EASY_TASKS = [
         },
         "expected_status": 200,
         "expected_schema": {"users": []},
+        "max_steps": 10,
     },
     {
         "id": "easy_content_type",
@@ -24,6 +25,7 @@ EASY_TASKS = [
         },
         "expected_status": 200,
         "expected_schema": {"item_id": "", "name": "", "created": True},
+        "max_steps": 10,
     },
     {
         "id": "easy_query_param",
@@ -37,5 +39,6 @@ EASY_TASKS = [
         },
         "expected_status": 200,
         "expected_schema": {"results": [], "total": 0},
+        "max_steps": 10,
     },
 ]

tasks/expert.py

@@ -15,17 +15,17 @@ EXPERT_TASKS = [
     },
     {
         "id": "expert_stateful_chain",
-        "description": "First, create an item with field '{dynamic_item_field}' set to 'temporary_item' via POST /mock_api/items. Then, safely clean it up by sending a DELETE request to /mock_api/items/{{item_id}}. The current request is a placeholder.",
+        "description": "First, create an item with field 'name' set to 'temporary_item' via POST /mock_api/items. Then, safely clean it up by sending a DELETE request to /mock_api/items/{item_id}. The current request is a placeholder.",
         "broken_request": {
             "method": "POST",
             "url": "/mock_api/items",
             "headers": {"Content-Type": "application/json"},
-            "body": {"{dynamic_item_field}": "temporary_item"},
+            "body": {"name": "temporary_item"},
             "query_params": {},
         },
         "expected_status": [200, 200],
         "expected_schema": [
-            {"item_id": "", "{dynamic_item_field}": "", "created": True},
+            {"item_id": "", "name": "", "created": True},
             {"status": "", "item_id": ""}
         ],
         "max_steps": 10,

tasks/medium.py

@@ -11,6 +11,7 @@ MEDIUM_TASKS = [
         },
         "expected_status": 200,
         "expected_schema": {"order_id": 0, "status": ""},
+        "max_steps": 10,
     },
     {
         "id": "medium_type_mismatch",
@@ -24,6 +25,7 @@ MEDIUM_TASKS = [
         },
         "expected_status": 200,
         "expected_schema": {"order_id": 0, "product_id": 0},
+        "max_steps": 10,
     },
     {
         "id": "medium_nested_field",
@@ -37,5 +39,6 @@ MEDIUM_TASKS = [
         },
         "expected_status": 200,
         "expected_schema": {"name": "", "address": {}, "updated": True},
+        "max_steps": 10,
     },
 ]

tests/test_environment.py

@@ -0,0 +1,248 @@
+"""
+Tests for API Debug Environment
+================================
+Covers:
+  1. Grader unit tests (grade + _schema_match + get_feedback)
+  2. Task registry validation (all tasks have required keys)
+  3. Mocked episode lifecycle (reset → step → done)
+
+Run with:
+    pytest tests/ -v
+"""
+
+import json
+import sys
+import os
+from unittest.mock import MagicMock, patch
+
+# Ensure project root is on sys.path so imports resolve without installation
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+import pytest
+
+
+# ---------------------------------------------------------------------------
+# 1. Grader unit tests
+# ---------------------------------------------------------------------------
+
+class TestAPIGrader:
+    """Unit tests for APIGrader.grade(), ._schema_match(), .get_feedback()."""
+
+    @pytest.fixture(autouse=True)
+    def grader(self):
+        from graders.grader import APIGrader
+        self.g = APIGrader()
+
+    # --- _schema_match ---
+
+    def test_schema_match_empty_schema_returns_1(self):
+        assert self.g._schema_match({"foo": "bar"}, {}) == 1.0
+
+    def test_schema_match_full_match(self):
+        body   = {"id": 1, "name": "test", "active": True}
+        schema = {"id": 0,  "name": ""}
+        assert self.g._schema_match(body, schema) == 1.0
+
+    def test_schema_match_partial_match(self):
+        body   = {"id": 1, "name": "test"}
+        schema = {"id": 0, "name": "", "missing_key": "x"}
+        score  = self.g._schema_match(body, schema)
+        # 2 out of 3 keys match
+        assert abs(score - 2/3) < 1e-9
+
+    def test_schema_match_type_mismatch(self):
+        body   = {"id": "string_not_int"}
+        schema = {"id": 0}
+        assert self.g._schema_match(body, schema) == 0.0
+
+    # --- grade ---
+
+    def test_grade_wrong_status_returns_partial(self):
+        score = self.g.grade(
+            response_status=404, response_body="not found",
+            expected_status=200, expected_schema={"id": 0},
+            attempt=1, max_steps=5,
+        )
+        assert 0.001 <= score <= 0.999
+        assert score < 0.5   # should be a small partial score
+
+    def test_grade_correct_status_non_json_body(self):
+        score = self.g.grade(
+            response_status=200, response_body="not json!",
+            expected_status=200, expected_schema={"id": 0},
+            attempt=1, max_steps=5,
+        )
+        assert score == 0.25
+
+    def test_grade_correct_status_perfect_schema(self):
+        body_dict = {"id": 1, "name": "alice"}
+        score = self.g.grade(
+            response_status=200,
+            response_body=json.dumps(body_dict),
+            expected_status=200,
+            expected_schema={"id": 0, "name": ""},
+            attempt=1, max_steps=5,
+        )
+        assert score >= 0.8
+
+    def test_grade_score_clamped(self):
+        # Even with a perfect response the score must be < 1.0
+        body_dict = {"status": "ok"}
+        score = self.g.grade(
+            response_status=200,
+            response_body=json.dumps(body_dict),
+            expected_status=200,
+            expected_schema={"status": ""},
+            attempt=1, max_steps=5,
+        )
+        assert 0.001 <= score <= 0.999
+
+    # --- get_feedback ---
+
+    def test_feedback_correct_status(self):
+        msg = self.g.get_feedback(200, 200)
+        assert "Correct" in msg
+
+    def test_feedback_401(self):
+        msg = self.g.get_feedback(401, 200)
+        assert "auth" in msg.lower() or "401" in msg
+
+    def test_feedback_unknown_status(self):
+        msg = self.g.get_feedback(418, 200)
+        assert "418" in msg
+
+
+# ---------------------------------------------------------------------------
+# 2. Task registry validation
+# ---------------------------------------------------------------------------
+
+class TestTaskRegistry:
+    """Validates that every task in the registry has the required keys."""
+
+    REQUIRED_KEYS = {
+        "id", "description", "broken_request",
+        "expected_status", "expected_schema", "max_steps",
+    }
+
+    REQUIRED_REQUEST_KEYS = {"method", "url", "headers", "body", "query_params"}
+
+    @pytest.fixture(autouse=True)
+    def registry(self):
+        from tasks.registry import TASK_REGISTRY
+        self.registry = TASK_REGISTRY
+
+    def test_all_difficulty_levels_present(self):
+        for level in ("easy", "medium", "hard", "expert"):
+            assert level in self.registry, f"Missing difficulty level: {level}"
+
+    def test_each_difficulty_has_at_least_one_task(self):
+        for level, tasks in self.registry.items():
+            assert len(tasks) >= 1, f"No tasks defined for level: {level}"
+
+    def test_all_tasks_have_required_keys(self):
+        for level, tasks in self.registry.items():
+            for task in tasks:
+                missing = self.REQUIRED_KEYS - set(task.keys())
+                assert not missing, (
+                    f"Task '{task.get('id', '?')}' in '{level}' is missing keys: {missing}"
+                )
+
+    def test_broken_request_has_required_keys(self):
+        for level, tasks in self.registry.items():
+            for task in tasks:
+                req     = task["broken_request"]
+                missing = self.REQUIRED_REQUEST_KEYS - set(req.keys())
+                assert not missing, (
+                    f"Task '{task.get('id', '?')}' broken_request missing: {missing}"
+                )
+
+    def test_no_format_placeholder_keys_in_body(self):
+        """Ensure no task body contains un-formatted {placeholder} as dict keys."""
+        for level, tasks in self.registry.items():
+            for task in tasks:
+                body = task["broken_request"].get("body") or {}
+                for key in body:
+                    assert not (key.startswith("{") and key.endswith("}")), (
+                        f"Task '{task.get('id', '?')}' has placeholder key in body: {key!r}"
+                    )
+
+    def test_max_steps_positive_int(self):
+        for level, tasks in self.registry.items():
+            for task in tasks:
+                assert isinstance(task["max_steps"], int) and task["max_steps"] > 0
+
+
+# ---------------------------------------------------------------------------
+# 3. Mocked episode lifecycle
+# ---------------------------------------------------------------------------
+
+class TestEpisodeLifecycle:
+    """Tests the reset → step → done lifecycle using a mocked httpx client."""
+
+    def _make_env(self, mock_httpx_client):
+        """Build an APIDebugEnvironment with httpx patched out."""
+        # Patch httpx.Client used in api_debug_environment
+        with patch("httpx.Client") as MockClient:
+            cm = MockClient.return_value.__enter__.return_value
+            cm.post.return_value = MagicMock(status_code=200, text="{}", headers={})
+            cm.request.return_value = MagicMock(
+                status_code=200,
+                text='{"id": 1, "name": "alice"}',
+                headers={"content-type": "application/json"},
+            )
+            import importlib
+            import server.api_debug_environment as ade_module
+            importlib.reload(ade_module)
+            env = ade_module.APIDebugEnvironment()
+        return env
+
+    def test_reset_returns_observation(self):
+        with patch("httpx.Client") as MockClient:
+            cm = MockClient.return_value.__enter__.return_value
+            cm.post.return_value = MagicMock(status_code=200, text="{}", headers={})
+            from server.api_debug_environment import APIDebugEnvironment
+            env = APIDebugEnvironment()
+            obs = env.reset(task_id="easy")
+            assert obs is not None
+            assert hasattr(obs, "task_description")
+            assert not obs.done
+
+    def test_step_increments_step_count(self):
+        with patch("httpx.Client") as MockClient:
+            cm = MockClient.return_value.__enter__.return_value
+            cm.post.return_value = MagicMock(status_code=200, text="{}", headers={})
+            cm.request.return_value = MagicMock(
+                status_code=200,
+                text='{"id": 1, "name": "alice"}',
+                headers={"content-type": "application/json"},
+            )
+            from server.api_debug_environment import APIDebugEnvironment
+            from models import APIAction
+            env = APIDebugEnvironment()
+            env.reset(task_id="easy")
+            action = APIAction(method="GET", url="/mock_api/users",
+                               headers={}, body={}, query_params={})
+            obs = env.step(action)
+            assert env.state.step_count == 1
+            assert obs.reward >= 0.001
+
+    def test_done_when_max_steps_exceeded(self):
+        with patch("httpx.Client") as MockClient:
+            cm = MockClient.return_value.__enter__.return_value
+            cm.post.return_value = MagicMock(status_code=200, text="{}", headers={})
+            cm.request.return_value = MagicMock(
+                status_code=404,
+                text='{"detail":"not found"}',
+                headers={},
+            )
+            from server.api_debug_environment import APIDebugEnvironment
+            from models import APIAction
+            env = APIDebugEnvironment()
+            env.reset(task_id="easy")
+            action = APIAction(method="GET", url="/mock_api/wrong",
+                               headers={}, body={}, query_params={})
+            max_steps = env.state.max_steps
+            obs = None
+            for _ in range(max_steps):
+                obs = env.step(action)
+            assert obs.done