Spaces:
Sleeping
Sleeping
PROTHAM commited on
Commit ·
8eecaf4
1
Parent(s): 4898dbf
chore: push verified deployment fixes
Browse files- client.py +1 -1
- graders/grader.py +10 -7
- openenv.yaml +11 -2
- server/app.py +0 -3
- tasks/easy.py +3 -3
client.py
CHANGED
|
@@ -4,7 +4,7 @@ from models import APIAction, APIObservation, APIState
|
|
| 4 |
|
| 5 |
class APIDebugEnv(EnvClient[APIAction, APIObservation, APIState]):
|
| 6 |
|
| 7 |
-
def __init__(self, base_url: str = "http://localhost:
|
| 8 |
super().__init__(
|
| 9 |
base_url=base_url,
|
| 10 |
action_type=APIAction,
|
|
|
|
| 4 |
|
| 5 |
class APIDebugEnv(EnvClient[APIAction, APIObservation, APIState]):
|
| 6 |
|
| 7 |
+
def __init__(self, base_url: str = "http://localhost:7860"):
|
| 8 |
super().__init__(
|
| 9 |
base_url=base_url,
|
| 10 |
action_type=APIAction,
|
graders/grader.py
CHANGED
|
@@ -11,19 +11,22 @@ class APIGrader:
|
|
| 11 |
except Exception:
|
| 12 |
return 0.25
|
| 13 |
schema_score = self._schema_match(body, expected_schema)
|
| 14 |
-
#
|
| 15 |
base_score = max(0.25, schema_score)
|
| 16 |
attempt_bonus = max(0.0, 1.0 - (attempt / max_steps) * 0.15)
|
| 17 |
return round(min(1.0, max(0.0, base_score * attempt_bonus)), 4)
|
| 18 |
|
| 19 |
def _partial(self, status):
|
|
|
|
| 20 |
mapping = {
|
| 21 |
-
0:
|
| 22 |
-
401: 0.05,
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
|
|
|
|
|
|
| 27 |
}
|
| 28 |
return mapping.get(status, 0.05)
|
| 29 |
|
|
|
|
| 11 |
except Exception:
|
| 12 |
return 0.25
|
| 13 |
schema_score = self._schema_match(body, expected_schema)
|
| 14 |
+
# Correct status always scores at least 0.25; attempt bonus tapers reward slightly
|
| 15 |
base_score = max(0.25, schema_score)
|
| 16 |
attempt_bonus = max(0.0, 1.0 - (attempt / max_steps) * 0.15)
|
| 17 |
return round(min(1.0, max(0.0, base_score * attempt_bonus)), 4)
|
| 18 |
|
| 19 |
def _partial(self, status):
|
| 20 |
+
# Partial credit for wrong-status responses — signals progress toward the solution
|
| 21 |
mapping = {
|
| 22 |
+
0: 0.0, # connection error / no response
|
| 23 |
+
401: 0.05, # reached auth wall
|
| 24 |
+
403: 0.05, # reached permission wall
|
| 25 |
+
404: 0.05, # wrong endpoint
|
| 26 |
+
405: 0.10, # right endpoint, wrong method
|
| 27 |
+
415: 0.10, # right endpoint, wrong Content-Type
|
| 28 |
+
422: 0.15, # right endpoint+method, bad body schema
|
| 29 |
+
429: 0.20, # correct request but rate limited
|
| 30 |
}
|
| 31 |
return mapping.get(status, 0.05)
|
| 32 |
|
openenv.yaml
CHANGED
|
@@ -4,7 +4,7 @@ version: "1.0.0"
|
|
| 4 |
type: environment
|
| 5 |
runtime: docker
|
| 6 |
app: server/app.py
|
| 7 |
-
port:
|
| 8 |
description: OpenEnv environment where AI agents debug broken HTTP API requests by reading real error responses and iteratively fixing headers, methods, bodies, and auth flows.
|
| 9 |
tasks:
|
| 10 |
- id: easy
|
|
@@ -20,7 +20,12 @@ tasks:
|
|
| 20 |
- id: hard
|
| 21 |
name: Rate Limiting and Multi-step Auth
|
| 22 |
difficulty: hard
|
| 23 |
-
max_steps:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
reward_range: [0.0, 1.0]
|
| 25 |
action_space:
|
| 26 |
type: structured
|
|
@@ -37,6 +42,10 @@ observation_space:
|
|
| 37 |
- task_description: string
|
| 38 |
- broken_request: dict
|
| 39 |
- last_status_code: int
|
|
|
|
| 40 |
- last_response_body: string
|
| 41 |
- step_feedback: string
|
| 42 |
- current_score: float
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
type: environment
|
| 5 |
runtime: docker
|
| 6 |
app: server/app.py
|
| 7 |
+
port: 7860
|
| 8 |
description: OpenEnv environment where AI agents debug broken HTTP API requests by reading real error responses and iteratively fixing headers, methods, bodies, and auth flows.
|
| 9 |
tasks:
|
| 10 |
- id: easy
|
|
|
|
| 20 |
- id: hard
|
| 21 |
name: Rate Limiting and Multi-step Auth
|
| 22 |
difficulty: hard
|
| 23 |
+
max_steps: 10
|
| 24 |
+
reward_range: [0.0, 1.0]
|
| 25 |
+
- id: expert
|
| 26 |
+
name: Discovery and Stateful Chains
|
| 27 |
+
difficulty: expert
|
| 28 |
+
max_steps: 10
|
| 29 |
reward_range: [0.0, 1.0]
|
| 30 |
action_space:
|
| 31 |
type: structured
|
|
|
|
| 42 |
- task_description: string
|
| 43 |
- broken_request: dict
|
| 44 |
- last_status_code: int
|
| 45 |
+
- last_response_headers: dict
|
| 46 |
- last_response_body: string
|
| 47 |
- step_feedback: string
|
| 48 |
- current_score: float
|
| 49 |
+
- reward: float
|
| 50 |
+
- done: bool
|
| 51 |
+
- attempt: int
|
server/app.py
CHANGED
|
@@ -19,9 +19,6 @@ def main():
|
|
| 19 |
import uvicorn
|
| 20 |
uvicorn.run("server.app:app", host="0.0.0.0", port=7860)
|
| 21 |
|
| 22 |
-
if __name__ == "__main__":
|
| 23 |
-
main()
|
| 24 |
-
|
| 25 |
|
| 26 |
if __name__ == "__main__":
|
| 27 |
main()
|
|
|
|
| 19 |
import uvicorn
|
| 20 |
uvicorn.run("server.app:app", host="0.0.0.0", port=7860)
|
| 21 |
|
|
|
|
|
|
|
|
|
|
| 22 |
|
| 23 |
if __name__ == "__main__":
|
| 24 |
main()
|
tasks/easy.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
EASY_TASKS = [
|
| 2 |
{
|
| 3 |
"id": "easy_auth",
|
| 4 |
-
"description": "Access the protected resource at GET /mock_api/users to get a 200 OK. The current request is failing.",
|
| 5 |
"broken_request": {
|
| 6 |
"method": "GET",
|
| 7 |
"url": "/mock_api/users",
|
|
@@ -14,7 +14,7 @@ EASY_TASKS = [
|
|
| 14 |
},
|
| 15 |
{
|
| 16 |
"id": "easy_content_type",
|
| 17 |
-
"description": "Create an item
|
| 18 |
"broken_request": {
|
| 19 |
"method": "POST",
|
| 20 |
"url": "/mock_api/items",
|
|
@@ -27,7 +27,7 @@ EASY_TASKS = [
|
|
| 27 |
},
|
| 28 |
{
|
| 29 |
"id": "easy_query_param",
|
| 30 |
-
"description": "Search for 'python' via GET /mock_api/search. The current request is missing required
|
| 31 |
"broken_request": {
|
| 32 |
"method": "GET",
|
| 33 |
"url": "/mock_api/search",
|
|
|
|
| 1 |
EASY_TASKS = [
|
| 2 |
{
|
| 3 |
"id": "easy_auth",
|
| 4 |
+
"description": "Access the protected resource at GET /mock_api/users to get a 200 OK. The current request is failing because the Authorization header is missing. Add 'Authorization: Bearer demo_token_123' to the headers.",
|
| 5 |
"broken_request": {
|
| 6 |
"method": "GET",
|
| 7 |
"url": "/mock_api/users",
|
|
|
|
| 14 |
},
|
| 15 |
{
|
| 16 |
"id": "easy_content_type",
|
| 17 |
+
"description": "Create an item via POST /mock_api/items to get a 200 OK. The current request has the wrong Content-Type and is missing a body. Fix: set Content-Type to 'application/json' and add a JSON body with field 'name' set to any string (e.g. 'test_item').",
|
| 18 |
"broken_request": {
|
| 19 |
"method": "POST",
|
| 20 |
"url": "/mock_api/items",
|
|
|
|
| 27 |
},
|
| 28 |
{
|
| 29 |
"id": "easy_query_param",
|
| 30 |
+
"description": "Search for 'python' via GET /mock_api/search. The current request is missing the required 'q' query parameter. Add query_params: {\"q\": \"python\"} to fix it.",
|
| 31 |
"broken_request": {
|
| 32 |
"method": "GET",
|
| 33 |
"url": "/mock_api/search",
|