readme

api_debug_env.egg-info/PKG-INFO

@@ -0,0 +1,17 @@
+Metadata-Version: 2.4
+Name: api-debug-env
+Version: 1.0.0
+Summary: OpenEnv environment for AI-powered HTTP API debugging
+Requires-Python: >=3.10
+License-File: LICENSE
+Requires-Dist: openenv-core>=0.2.3
+Requires-Dist: fastapi>=0.104.0
+Requires-Dist: uvicorn>=0.24.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: openai>=1.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21; extra == "dev"
+Dynamic: license-file

api_debug_env.egg-info/SOURCES.txt

@@ -0,0 +1,21 @@
+LICENSE
+README.md
+pyproject.toml
+api_debug_env.egg-info/PKG-INFO
+api_debug_env.egg-info/SOURCES.txt
+api_debug_env.egg-info/dependency_links.txt
+api_debug_env.egg-info/entry_points.txt
+api_debug_env.egg-info/requires.txt
+api_debug_env.egg-info/top_level.txt
+graders/__init__.py
+graders/grader.py
+server/__init__.py
+server/api_debug_environment.py
+server/app.py
+server/mock_api.py
+tasks/__init__.py
+tasks/easy.py
+tasks/expert.py
+tasks/hard.py
+tasks/medium.py
+tasks/registry.py

api_debug_env.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

api_debug_env.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+server = server.app:main

api_debug_env.egg-info/requires.txt

@@ -0,0 +1,11 @@
+openenv-core>=0.2.3
+fastapi>=0.104.0
+uvicorn>=0.24.0
+pydantic>=2.0.0
+httpx>=0.27.0
+python-dotenv>=1.0.0
+openai>=1.0.0
+
+[dev]
+pytest>=7.0
+pytest-asyncio>=0.21

api_debug_env.egg-info/top_level.txt

@@ -0,0 +1,4 @@
+graders
+outputs
+server
+tasks

graders/grader.py

@@ -14,12 +14,13 @@ class APIGrader:
         # Correct status always scores at least 0.25; attempt bonus tapers reward slightly
         base_score = max(0.25, schema_score)
         attempt_bonus = max(0.0, 1.0 - (attempt / max_steps) * 0.15)
-        return round(min(1.0, max(0.0, base_score * attempt_bonus)), 4)
+        raw_score = base_score * attempt_bonus
+        return round(min(0.999, max(0.001, raw_score)), 4)
 
     def _partial(self, status):
         # Partial credit for wrong-status responses — signals progress toward the solution
         mapping = {
-            0:   0.0,   # connection error / no response
+            0:   0.001,   # connection error / no response
             401: 0.05,  # reached auth wall
             403: 0.05,  # reached permission wall
             404: 0.05,  # wrong endpoint
@@ -33,8 +34,17 @@ class APIGrader:
     def _schema_match(self, body, schema):
         if not schema:
             return 1.0
-        matched = sum(1 for k, v in schema.items() if k in body and type(body[k]) == type(v))
-        return matched / len(schema)
+            
+        schemas = schema if isinstance(schema, list) else [schema]
+        best_match = 0.0
+        
+        for s in schemas:
+            matched = sum(1 for k, v in s.items() if k in body and type(body[k]) == type(v))
+            score = matched / len(s) if len(s) > 0 else 1.0
+            if score > best_match:
+                best_match = score
+                
+        return best_match
 
     def get_feedback(self, status, expected_status):
         if status == expected_status:

inference.py

@@ -286,7 +286,7 @@ async def main() -> None:
             )
             # Do NOT return — we must still run tasks to emit required log lines
 
-        for task_id in ["easy", "medium", "hard"]:
+        for task_id in ["easy", "medium", "hard", "expert", "auto"]:
             await run_task(client, task_id)
 
 

models.py

@@ -31,9 +31,13 @@ class APIObservation(Observation):
     attempt: int = Field(default=0)
     reward: float = Field(default=0.0)
     done: bool = Field(default=False)
+    current_step_index: int = Field(default=0)
 
 
 class APIState(State):
     task_id: str = Field(default="easy")
     max_steps: int = Field(default=10)
     solved: bool = Field(default=False)
+    current_step_index: int = Field(default=0)
+    visited_endpoints: set = Field(default_factory=set)
+    curriculum_level: int = Field(default=0)

server/api_debug_environment.py

@@ -17,20 +17,59 @@ MOCK_BASE = "http://localhost:7860"
 class APIDebugEnvironment(Environment):
 
     def __init__(self):
-        self._state = APIState(episode_id="init", step_count=0, task_id="easy", max_steps=5, solved=False)
+        self._state = APIState(episode_id="init", step_count=0, task_id="easy", max_steps=10, solved=False, current_step_index=0, curriculum_level=0)
         self._current_task = None
 
-    def reset(self, task_id: str = "easy", **kwargs) -> APIObservation:
-        if task_id not in TASK_REGISTRY:
-            task_id = "easy"
-        self._current_task = random.choice(TASK_REGISTRY[task_id])
-        max_steps = self._current_task.get("max_steps", 5)
+    def reset(self, task_id: str = "auto", **kwargs) -> APIObservation:
+        import copy
+        import uuid
+
+        old_level = getattr(self._state, "curriculum_level", 0)
+        if getattr(self._state, "solved", False):
+            old_level += 1
+
+        effective_task_id = task_id
+        if task_id == "auto":
+            levels = ["easy", "medium", "hard", "expert"]
+            effective_task_id = levels[min(old_level, len(levels) - 1)]
+
+        if effective_task_id not in TASK_REGISTRY:
+            effective_task_id = "easy"
+            
+        task = copy.deepcopy(random.choice(TASK_REGISTRY[effective_task_id]))
+
+        dyn_config = {
+            "demo_token": f"tok_{uuid.uuid4().hex[:8]}",
+            "client_id": f"cli_{uuid.uuid4().hex[:8]}",
+            "client_secret": f"sec_{uuid.uuid4().hex[:8]}"
+        }
+
+        try:
+            with httpx.Client(base_url=MOCK_BASE, timeout=2.0) as http:
+                http.post("/mock_api/_admin/reset", json=dyn_config)
+        except Exception:
+            pass
+
+        try:
+            task["description"] = task["description"].format(**dyn_config)
+            req = task["broken_request"]
+            if "headers" in req:
+                req["headers"] = {k: (v.format(**dyn_config) if isinstance(v, str) else v) for k, v in req["headers"].items()}
+        except Exception:
+            pass
+
+        self._current_task = task
+        max_steps = self._current_task.get("max_steps", 10)
+        
         self._state = APIState(
             episode_id=f"ep_{random.randint(10000, 99999)}",
             step_count=0,
             task_id=task_id,
             max_steps=max_steps,
             solved=False,
+            current_step_index=0,
+            visited_endpoints=set(),
+            curriculum_level=old_level
         )
         return APIObservation(
             task_id=task_id,
@@ -40,15 +79,16 @@ class APIDebugEnvironment(Environment):
             last_response_headers={},
             last_response_body="",
             step_feedback="Episode started. Read the task description and fix the broken request.",
-            current_score=0.0,
+            current_score=0.001,
             attempt=0,
-            reward=0.0,
+            reward=0.001,
             done=False,
+            current_step_index=0
         )
 
     def step(self, action: APIAction, **kwargs) -> APIObservation:
         if self._current_task is None:
-            self.reset(task_id="easy")
+            self.reset(task_id="auto")
 
         self._state.step_count += 1
         task = self._current_task
@@ -70,19 +110,42 @@ class APIDebugEnvironment(Environment):
             resp_headers = {}
             resp_body = f"Request error: {str(e)}"
 
+        expected_schema = task["expected_schema"]
+        expected_status = task["expected_status"]
+        is_chain = isinstance(expected_schema, list)
+
+        target_schema = expected_schema[self._state.current_step_index] if is_chain else expected_schema
+        target_status = expected_status[self._state.current_step_index] if isinstance(expected_status, list) else expected_status
+
         reward = GRADER.grade(
             response_status=status,
             response_body=resp_body,
-            expected_status=task["expected_status"],
-            expected_schema=task["expected_schema"],
+            expected_status=target_status,
+            expected_schema=target_schema,
             attempt=self._state.step_count,
             max_steps=self._state.max_steps,
         )
 
-        feedback = GRADER.get_feedback(status, task["expected_status"])
-        solved = status == task["expected_status"] and reward >= 0.8
-        self._state.solved = solved
-        done = solved or self._state.step_count >= self._state.max_steps
+        feedback = GRADER.get_feedback(status, target_status)
+        step_solved = status == target_status and reward >= 0.8
+        
+        if is_chain and step_solved:
+            self._state.current_step_index += 1
+            if self._state.current_step_index >= len(expected_schema):
+                self._state.solved = True
+                feedback += " Chain completed successfully!"
+            else:
+                feedback += f" Step {self._state.current_step_index} complete. Moving to next step."
+                reward = 0.5  # Partial reward for finishing a step
+        else:
+            self._state.solved = step_solved if not is_chain else False
+
+        if action.url not in self._state.visited_endpoints and status > 0 and status < 500:
+            self._state.visited_endpoints.add(action.url)
+            reward += 0.05
+        reward = round(min(0.999, max(0.001, reward)), 4)
+
+        done = self._state.solved or self._state.step_count >= self._state.max_steps
 
         return APIObservation(
             task_id=self._state.task_id,
@@ -96,6 +159,7 @@ class APIDebugEnvironment(Environment):
             attempt=self._state.step_count,
             reward=reward,
             done=done,
+            current_step_index=self._state.current_step_index
         )
 
     @property

server/mock_api.py

@@ -10,6 +10,13 @@ _issued_tokens = set()
 _request_log = {}
 _items_db = {}
 
+DYNAMIC_CONFIG = {
+    "demo_token": "demo_token_123",
+    "client_id": "abc",
+    "client_secret": "xyz",
+    "dynamic_item_field": "name"
+}
+
 LOG_PAGES = {
     None: {"items": list(range(10)), "next_cursor": "cur_abc", "has_more": True},
     "cur_abc": {"items": list(range(10, 20)), "next_cursor": "cur_def", "has_more": True},
@@ -17,10 +24,12 @@ LOG_PAGES = {
 }
 
 @router.post("/_admin/reset")
-async def admin_reset():
+async def admin_reset(config: Optional[dict] = None):
     _issued_tokens.clear()
     _request_log.clear()
     _items_db.clear()
+    if config:
+        DYNAMIC_CONFIG.update(config)
     return {"status": "reset_successful"}
 
 
@@ -30,7 +39,7 @@ class TokenRequest(BaseModel):
 
 @router.post("/auth/token")
 async def get_token(body: TokenRequest):
-    if body.client_id == "abc" and body.client_secret == "xyz":
+    if body.client_id == DYNAMIC_CONFIG["client_id"] and body.client_secret == DYNAMIC_CONFIG["client_secret"]:
         token = f"tok_{int(time.time())}"
         _issued_tokens.add(token)
         return {"access_token": token, "expires_in": 60}
@@ -39,11 +48,12 @@ async def get_token(body: TokenRequest):
 
 @router.get("/users")
 async def get_users(authorization: Optional[str] = Header(default=None)):
+    dt = DYNAMIC_CONFIG["demo_token"]
     if not authorization or not authorization.startswith("Bearer "):
-        raise HTTPException(status_code=401, detail="Authorization header missing or malformed. Expected format: Bearer <token>. Use 'demo_token_123' for testing.")
+        raise HTTPException(status_code=401, detail=f"Authorization header missing or malformed. Expected format: Bearer <token>. Use '{dt}' for testing.")
     token = authorization.split(" ")[1]
-    if token not in _issued_tokens and token != "demo_token_123":
-        raise HTTPException(status_code=401, detail="Invalid token. Try 'demo_token_123'.")
+    if token not in _issued_tokens and token != dt:
+        raise HTTPException(status_code=401, detail=f"Invalid token. Try '{dt}'.")
     return {"users": [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}]}
 
 
@@ -53,11 +63,12 @@ async def create_item(request: Request):
     if "application/json" not in ct:
         raise HTTPException(status_code=415, detail="Content-Type must be application/json")
     body = await request.json()
-    if "name" not in body:
-        raise HTTPException(status_code=422, detail=[{"loc": ["body", "name"], "msg": "field required"}])
+    field = DYNAMIC_CONFIG.get("dynamic_item_field", "name")
+    if field not in body:
+        raise HTTPException(status_code=422, detail=[{"loc": ["body", field], "msg": "field required"}])
     item_id = str(uuid.uuid4())
-    _items_db[item_id] = body["name"]
-    return {"item_id": item_id, "name": body["name"], "created": True}
+    _items_db[item_id] = body[field]
+    return {"item_id": item_id, "name": body[field], "created": True}
 
 @router.delete("/items/{item_id}")
 async def delete_item(item_id: str):

tasks/easy.py

@@ -1,7 +1,7 @@
 EASY_TASKS = [
     {
         "id": "easy_auth",
-        "description": "Access the protected resource at GET /mock_api/users to get a 200 OK. The current request is failing because the Authorization header is missing. Add 'Authorization: Bearer demo_token_123' to the headers.",
+        "description": "Access the protected resource at GET /mock_api/users to get a 200 OK. The current request is failing because the Authorization header is missing. Add 'Authorization: Bearer {demo_token}' to the headers.",
         "broken_request": {
             "method": "GET",
             "url": "/mock_api/users",

tasks/expert.py

@@ -15,16 +15,19 @@ EXPERT_TASKS = [
     },
     {
         "id": "expert_stateful_chain",
-        "description": "First, create an item with name 'temporary_item' via POST /mock_api/items to get an auto-generated item_id. Then, safely clean it up by sending a DELETE request to /mock_api/items/{item_id}. The current request is a placeholder.",
+        "description": "First, create an item with field '{dynamic_item_field}' set to 'temporary_item' via POST /mock_api/items. Then, safely clean it up by sending a DELETE request to /mock_api/items/{{item_id}}. The current request is a placeholder.",
         "broken_request": {
             "method": "POST",
             "url": "/mock_api/items",
             "headers": {"Content-Type": "application/json"},
-            "body": {"name": "temporary_item"},
+            "body": {"{dynamic_item_field}": "temporary_item"},
             "query_params": {},
         },
-        "expected_status": 200,
-        "expected_schema": {"status": "", "item_id": ""},
+        "expected_status": [200, 200],
+        "expected_schema": [
+            {"item_id": "", "{dynamic_item_field}": "", "created": True},
+            {"status": "", "item_id": ""}
+        ],
         "max_steps": 10,
     },
 ]

tasks/hard.py

@@ -3,7 +3,7 @@ HARD_TASKS = [
         "id": "hard_token_exchange",
         "description": (
             "The protected endpoint needs a fresh token. "
-            "Step 1: POST to /mock_api/auth/token with JSON body {\"client_id\": \"abc\", \"client_secret\": \"xyz\"} "
+            "Step 1: POST to /mock_api/auth/token with JSON body {{\"client_id\": \"{client_id}\", \"client_secret\": \"{client_secret}\"}} "
             "to get an access_token from the response. "
             "Step 2: Use that token as Authorization: Bearer <token> to GET /mock_api/protected. "
             "The token in the broken request is expired and will return 401."