PROTHAM commited on
Commit
8eecaf4
·
1 Parent(s): 4898dbf

chore: push verified deployment fixes

Browse files
Files changed (5) hide show
  1. client.py +1 -1
  2. graders/grader.py +10 -7
  3. openenv.yaml +11 -2
  4. server/app.py +0 -3
  5. tasks/easy.py +3 -3
client.py CHANGED
@@ -4,7 +4,7 @@ from models import APIAction, APIObservation, APIState
4
 
5
  class APIDebugEnv(EnvClient[APIAction, APIObservation, APIState]):
6
 
7
- def __init__(self, base_url: str = "http://localhost:8000"):
8
  super().__init__(
9
  base_url=base_url,
10
  action_type=APIAction,
 
4
 
5
  class APIDebugEnv(EnvClient[APIAction, APIObservation, APIState]):
6
 
7
+ def __init__(self, base_url: str = "http://localhost:7860"):
8
  super().__init__(
9
  base_url=base_url,
10
  action_type=APIAction,
graders/grader.py CHANGED
@@ -11,19 +11,22 @@ class APIGrader:
11
  except Exception:
12
  return 0.25
13
  schema_score = self._schema_match(body, expected_schema)
14
- # Ensure correct status always scores higher than any error status
15
  base_score = max(0.25, schema_score)
16
  attempt_bonus = max(0.0, 1.0 - (attempt / max_steps) * 0.15)
17
  return round(min(1.0, max(0.0, base_score * attempt_bonus)), 4)
18
 
19
  def _partial(self, status):
 
20
  mapping = {
21
- 0: 0.0,
22
- 401: 0.05, 403: 0.05, 404: 0.05,
23
- 405: 0.10, 415: 0.10,
24
- 422: 0.15,
25
- 429: 0.20,
26
- 200: 0.70,
 
 
27
  }
28
  return mapping.get(status, 0.05)
29
 
 
11
  except Exception:
12
  return 0.25
13
  schema_score = self._schema_match(body, expected_schema)
14
+ # Correct status always scores at least 0.25; attempt bonus tapers reward slightly
15
  base_score = max(0.25, schema_score)
16
  attempt_bonus = max(0.0, 1.0 - (attempt / max_steps) * 0.15)
17
  return round(min(1.0, max(0.0, base_score * attempt_bonus)), 4)
18
 
19
  def _partial(self, status):
20
+ # Partial credit for wrong-status responses — signals progress toward the solution
21
  mapping = {
22
+ 0: 0.0, # connection error / no response
23
+ 401: 0.05, # reached auth wall
24
+ 403: 0.05, # reached permission wall
25
+ 404: 0.05, # wrong endpoint
26
+ 405: 0.10, # right endpoint, wrong method
27
+ 415: 0.10, # right endpoint, wrong Content-Type
28
+ 422: 0.15, # right endpoint+method, bad body schema
29
+ 429: 0.20, # correct request but rate limited
30
  }
31
  return mapping.get(status, 0.05)
32
 
openenv.yaml CHANGED
@@ -4,7 +4,7 @@ version: "1.0.0"
4
  type: environment
5
  runtime: docker
6
  app: server/app.py
7
- port: 8000
8
  description: OpenEnv environment where AI agents debug broken HTTP API requests by reading real error responses and iteratively fixing headers, methods, bodies, and auth flows.
9
  tasks:
10
  - id: easy
@@ -20,7 +20,12 @@ tasks:
20
  - id: hard
21
  name: Rate Limiting and Multi-step Auth
22
  difficulty: hard
23
- max_steps: 5
 
 
 
 
 
24
  reward_range: [0.0, 1.0]
25
  action_space:
26
  type: structured
@@ -37,6 +42,10 @@ observation_space:
37
  - task_description: string
38
  - broken_request: dict
39
  - last_status_code: int
 
40
  - last_response_body: string
41
  - step_feedback: string
42
  - current_score: float
 
 
 
 
4
  type: environment
5
  runtime: docker
6
  app: server/app.py
7
+ port: 7860
8
  description: OpenEnv environment where AI agents debug broken HTTP API requests by reading real error responses and iteratively fixing headers, methods, bodies, and auth flows.
9
  tasks:
10
  - id: easy
 
20
  - id: hard
21
  name: Rate Limiting and Multi-step Auth
22
  difficulty: hard
23
+ max_steps: 10
24
+ reward_range: [0.0, 1.0]
25
+ - id: expert
26
+ name: Discovery and Stateful Chains
27
+ difficulty: expert
28
+ max_steps: 10
29
  reward_range: [0.0, 1.0]
30
  action_space:
31
  type: structured
 
42
  - task_description: string
43
  - broken_request: dict
44
  - last_status_code: int
45
+ - last_response_headers: dict
46
  - last_response_body: string
47
  - step_feedback: string
48
  - current_score: float
49
+ - reward: float
50
+ - done: bool
51
+ - attempt: int
server/app.py CHANGED
@@ -19,9 +19,6 @@ def main():
19
  import uvicorn
20
  uvicorn.run("server.app:app", host="0.0.0.0", port=7860)
21
 
22
- if __name__ == "__main__":
23
- main()
24
-
25
 
26
  if __name__ == "__main__":
27
  main()
 
19
  import uvicorn
20
  uvicorn.run("server.app:app", host="0.0.0.0", port=7860)
21
 
 
 
 
22
 
23
  if __name__ == "__main__":
24
  main()
tasks/easy.py CHANGED
@@ -1,7 +1,7 @@
1
  EASY_TASKS = [
2
  {
3
  "id": "easy_auth",
4
- "description": "Access the protected resource at GET /mock_api/users to get a 200 OK. The current request is failing.",
5
  "broken_request": {
6
  "method": "GET",
7
  "url": "/mock_api/users",
@@ -14,7 +14,7 @@ EASY_TASKS = [
14
  },
15
  {
16
  "id": "easy_content_type",
17
- "description": "Create an item with name 'test_item' via POST /mock_api/items to get a 200 OK. The current request payload is rejected by the server.",
18
  "broken_request": {
19
  "method": "POST",
20
  "url": "/mock_api/items",
@@ -27,7 +27,7 @@ EASY_TASKS = [
27
  },
28
  {
29
  "id": "easy_query_param",
30
- "description": "Search for 'python' via GET /mock_api/search. The current request is missing required parameters.",
31
  "broken_request": {
32
  "method": "GET",
33
  "url": "/mock_api/search",
 
1
  EASY_TASKS = [
2
  {
3
  "id": "easy_auth",
4
+ "description": "Access the protected resource at GET /mock_api/users to get a 200 OK. The current request is failing because the Authorization header is missing. Add 'Authorization: Bearer demo_token_123' to the headers.",
5
  "broken_request": {
6
  "method": "GET",
7
  "url": "/mock_api/users",
 
14
  },
15
  {
16
  "id": "easy_content_type",
17
+ "description": "Create an item via POST /mock_api/items to get a 200 OK. The current request has the wrong Content-Type and is missing a body. Fix: set Content-Type to 'application/json' and add a JSON body with field 'name' set to any string (e.g. 'test_item').",
18
  "broken_request": {
19
  "method": "POST",
20
  "url": "/mock_api/items",
 
27
  },
28
  {
29
  "id": "easy_query_param",
30
+ "description": "Search for 'python' via GET /mock_api/search. The current request is missing the required 'q' query parameter. Add query_params: {\"q\": \"python\"} to fix it.",
31
  "broken_request": {
32
  "method": "GET",
33
  "url": "/mock_api/search",