Add Expert tier tasks to evaluate multi-step UUID stateful tracking and OpenAPI discovery

inference.py

@@ -138,7 +138,7 @@ async def run_task(env, task_id):
 
 async def main():
     async with APIDebugEnv(base_url=ENV_URL) as env:
-        for task_id in ["easy", "medium", "hard"]:
+        for task_id in ["easy", "medium", "hard", "expert"]:
             await run_task(env, task_id)
 
 

server/mock_api.py

@@ -1,4 +1,5 @@
 import time
+import uuid
 from fastapi import APIRouter, Header, HTTPException, Request, Query
 from pydantic import BaseModel
 from typing import Optional
@@ -7,6 +8,7 @@ router = APIRouter(prefix="/mock_api", tags=["mock"])
 
 _issued_tokens = set()
 _request_log = {}
+_items_db = {}
 
 LOG_PAGES = {
     None: {"items": list(range(10)), "next_cursor": "cur_abc", "has_more": True},
@@ -18,6 +20,7 @@ LOG_PAGES = {
 async def admin_reset():
     _issued_tokens.clear()
     _request_log.clear()
+    _items_db.clear()
     return {"status": "reset_successful"}
 
 
@@ -52,7 +55,16 @@ async def create_item(request: Request):
     body = await request.json()
     if "name" not in body:
         raise HTTPException(status_code=422, detail=[{"loc": ["body", "name"], "msg": "field required"}])
-    return {"item_id": 42, "name": body["name"], "created": True}
+    item_id = str(uuid.uuid4())
+    _items_db[item_id] = body["name"]
+    return {"item_id": item_id, "name": body["name"], "created": True}
+
+@router.delete("/items/{item_id}")
+async def delete_item(item_id: str):
+    if item_id not in _items_db:
+        raise HTTPException(status_code=404, detail="Item not found")
+    del _items_db[item_id]
+    return {"status": "deleted", "item_id": item_id}
 
 
 @router.get("/search")
@@ -119,3 +131,11 @@ async def get_logs(cursor: Optional[str] = Query(default=None)):
     if cursor not in LOG_PAGES:
         raise HTTPException(status_code=400, detail=f"Invalid cursor: {cursor}")
     return LOG_PAGES[cursor]
+
+class MetricsBody(BaseModel):
+    cpu_load: float
+    memory_usage: float
+
+@router.post("/system/metrics", description="Ingest system metrics. Required JSON fields: cpu_load (float), memory_usage (float).")
+async def ingest_metrics(metrics: MetricsBody):
+    return {"status": "metrics_logged", "cpu": metrics.cpu_load, "mem": metrics.memory_usage}

tasks/easy.py

@@ -23,7 +23,7 @@ EASY_TASKS = [
             "query_params": {},
         },
         "expected_status": 200,
-        "expected_schema": {"item_id": 0, "name": "", "created": True},
+        "expected_schema": {"item_id": "", "name": "", "created": True},
     },
     {
         "id": "easy_query_param",

tasks/expert.py

@@ -0,0 +1,28 @@
+EXPERT_TASKS = [
+    {
+        "id": "expert_openapi_discovery",
+        "description": "Log system metrics. We forgot the exact endpoint path and required schema. Check the server documentation at /openapi.json to discover the metrics ingestion endpoint under /mock_api, then send a valid POST payload with cpu_load and memory_usage.",
+        "broken_request": {
+            "method": "POST",
+            "url": "/mock_api/unknown_metrics_endpoint",
+            "headers": {"Content-Type": "application/json"},
+            "body": {},
+            "query_params": {},
+        },
+        "expected_status": 200,
+        "expected_schema": {"status": "", "cpu": 0.0, "mem": 0.0},
+    },
+    {
+        "id": "expert_stateful_chain",
+        "description": "First, create an item with name 'temporary_item' via POST /mock_api/items to get an auto-generated item_id. Then, safely clean it up by sending a DELETE request to /mock_api/items/{item_id}. The current request is a placeholder.",
+        "broken_request": {
+            "method": "POST",
+            "url": "/mock_api/items",
+            "headers": {"Content-Type": "application/json"},
+            "body": {"name": "temporary_item"},
+            "query_params": {},
+        },
+        "expected_status": 200,
+        "expected_schema": {"status": "", "item_id": ""},
+    },
+]

tasks/registry.py

@@ -1,9 +1,11 @@
 from tasks.easy import EASY_TASKS
 from tasks.medium import MEDIUM_TASKS
 from tasks.hard import HARD_TASKS
+from tasks.expert import EXPERT_TASKS
 
 TASK_REGISTRY = {
     "easy": EASY_TASKS,
     "medium": MEDIUM_TASKS,
     "hard": HARD_TASKS,
+    "expert": EXPERT_TASKS,
 }