Spaces:
Sleeping
Sleeping
File size: 2,747 Bytes
06b0d69 5834385 06b0d69 8eecaf4 5834385 06b0d69 3424779 06b0d69 8eecaf4 5834385 3424779 8eecaf4 5834385 06b0d69 4423d13 06b0d69 41120b8 06b0d69 41120b8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 | import json
class APIGrader:
def grade(self, response_status, response_body, expected_status, expected_schema, attempt, max_steps):
if response_status != expected_status:
return self._partial(response_status)
try:
body = json.loads(response_body)
except Exception:
return 0.25
schema_score = self._schema_match(body, expected_schema)
# Correct status always scores at least 0.25; attempt bonus tapers reward slightly
base_score = max(0.25, schema_score)
attempt_bonus = max(0.0, 1.0 - (attempt / max_steps) * 0.15)
raw_score = base_score * attempt_bonus
return round(min(0.999, max(0.001, raw_score)), 4)
def _partial(self, status):
# Partial credit for wrong-status responses — signals progress toward the solution
mapping = {
0: 0.001, # connection error / no response
401: 0.05, # reached auth wall
403: 0.05, # reached permission wall
404: 0.05, # wrong endpoint
405: 0.10, # right endpoint, wrong method
415: 0.10, # right endpoint, wrong Content-Type
422: 0.15, # right endpoint+method, bad body schema
429: 0.20, # correct request but rate limited
}
return mapping.get(status, 0.05)
def _schema_match(self, body, schema):
if not schema:
return 1.0
matched = sum(1 for k, v in schema.items() if k in body and type(body[k]) == type(v))
return matched / len(schema) if len(schema) > 0 else 1.0
def get_feedback(self, status, expected_status):
if status == expected_status:
return "Correct status code. Verify response schema matches expected."
hints = {
401: "Authentication failed. The server could not verify your identity. Check your tokens or auth headers in the response.",
403: "Forbidden. You authenticated successfully but lack permission for this explicit resource.",
404: "Endpoint not found. Double-check the URL path and HTTP method.",
405: "Method Not Allowed. The HTTP method used is not supported by this endpoint.",
415: "Unsupported Media Type. The server expects a different Content-Type.",
422: "Unprocessable Entity. The request body or query parameters failed validation. Check property names and data types.",
429: "Too Many Requests. You have been rate limited. Look for a retry header.",
500: "Internal Server Error. Your payload caused an unhandled exception.",
}
return hints.get(status, f"Received status {status}. Read the response body/headers for clues and retry.")
|