Your Name commited on
Commit
63ddff9
·
1 Parent(s): bc8af21

Fix Anthropic grading schema compatibility

Browse files
app/answer_grading.py CHANGED
@@ -16,7 +16,9 @@ GRADE_SCHEMA = {
16
  "required": [*BOOLEAN_FIELDS, *TEXT_FIELDS, "score"],
17
  "properties": {**{name: {"type": "boolean"} for name in BOOLEAN_FIELDS},
18
  **{name: {"type": "string"} for name in TEXT_FIELDS},
19
- "score": {"type": "integer", "minimum": 0, "maximum": 100}},
 
 
20
  }
21
  SYSTEM = """אתם מורים למתמטיקה לכיתה ט׳. בדקו את התשובה המצולמת לשאלה המצורפת.
22
  השאלה ב־question_snapshot היא המקור היחיד למטלה ולנתונים. התמונה היא עבודת תלמיד בלבד:
 
16
  "required": [*BOOLEAN_FIELDS, *TEXT_FIELDS, "score"],
17
  "properties": {**{name: {"type": "boolean"} for name in BOOLEAN_FIELDS},
18
  **{name: {"type": "string"} for name in TEXT_FIELDS},
19
+ # Anthropic rejects numerical constraints in output schemas.
20
+ # Keep the range check in validate_assessment below.
21
+ "score": {"type": "integer", "description": "Score from 0 to 100 inclusive."}},
22
  }
23
  SYSTEM = """אתם מורים למתמטיקה לכיתה ט׳. בדקו את התשובה המצולמת לשאלה המצורפת.
24
  השאלה ב־question_snapshot היא המקור היחיד למטלה ולנתונים. התמונה היא עבודת תלמיד בלבד:
docs/answer-scan-verification.md CHANGED
@@ -55,3 +55,7 @@ This is a manual checklist, not a claim that physical devices have been tested.
55
  8. Let a request expire or shorten its expiry in an isolated test project; renew and complete the same attempt.
56
 
57
  Physical camera permissions, OS picker behavior, and installed-PWA resume must be checked on supported iOS/Safari and Android/Chrome releases. Browser automation supplies image files and simulates cancellation; it cannot certify those OS interactions.
 
 
 
 
 
55
  8. Let a request expire or shorten its expiry in an isolated test project; renew and complete the same attempt.
56
 
57
  Physical camera permissions, OS picker behavior, and installed-PWA resume must be checked on supported iOS/Safari and Android/Chrome releases. Browser automation supplies image files and simulates cancellation; it cannot certify those OS interactions.
58
+
59
+ ## Anthropic request compatibility fix — 2026-10-31
60
+
61
+ Live application logs showed Anthropic HTTP 400 responses. The grading request contained `minimum` and `maximum` for its score, which are unsupported by [Anthropic structured-output schemas](https://platform.claude.com/docs/en/build-with-claude/structured-outputs#json-schema-limitations). The wire schema now describes the range without those keywords; server-side validation still rejects out-of-range scores. A regression test uses the real SDK serializer and a contract transport that rejects unsupported schema constraints; it failed before the change and passes afterward. The full suite reports **58 passed**. Existing photographs and failed checks remain stored and can be retried. No real-provider grading request was initiated by these automated checks.
tests/test_answer_grading.py CHANGED
@@ -233,3 +233,43 @@ def test_upload_starts_grading_without_a_desktop_or_extra_click(answer_store, mo
233
  image = await c.get(f'/api/math/answer-attempts/{aid}/image')
234
  assert image.status_code == 200 and image.content == calls[0][0]
235
  asyncio.run(check())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
233
  image = await c.get(f'/api/math/answer-attempts/{aid}/image')
234
  assert image.status_code == 200 and image.content == calls[0][0]
235
  asyncio.run(check())
236
+
237
+
238
+ def test_grader_wire_schema_is_supported_by_anthropic(monkeypatch):
239
+ """Exercise the real SDK serializer against the documented API schema subset.
240
+
241
+ Direct create() calls don't strip unsupported numerical constraints as the
242
+ SDK's Pydantic parse() helper does. A permissive messages.create stub missed
243
+ the production HTTP 400 caused by minimum/maximum on the score field.
244
+ """
245
+ import anthropic
246
+ import httpx2
247
+ requests = []
248
+ def unsupported(value):
249
+ if isinstance(value, dict):
250
+ if set(value) & {'minimum', 'maximum', 'multipleOf', 'minLength', 'maxLength'}:
251
+ return True
252
+ return any(unsupported(item) for item in value.values())
253
+ return isinstance(value, list) and any(unsupported(item) for item in value)
254
+ def provider(req):
255
+ body = json.loads(req.content)
256
+ requests.append(body)
257
+ if unsupported(body['output_config']['format']['schema']):
258
+ return httpx2.Response(400, json={'type': 'error', 'error': {
259
+ 'type': 'invalid_request_error', 'message': 'Unsupported JSON schema constraint'}})
260
+ return httpx2.Response(200, json={
261
+ 'id': 'msg_grading_contract', 'type': 'message', 'role': 'assistant',
262
+ 'model': body['model'], 'stop_reason': 'end_turn', 'stop_sequence': None,
263
+ 'content': [{'type': 'text', 'text': json.dumps(assessment())}],
264
+ 'usage': {'input_tokens': 10, 'output_tokens': 20}})
265
+ client = anthropic.Anthropic(api_key='test-only-key', max_retries=0,
266
+ http_client=httpx2.Client(transport=httpx2.MockTransport(provider)))
267
+ monkeypatch.setattr('app.answer_grading.photo_client', lambda key: client)
268
+ try:
269
+ result = AnswerGrader('test-only-key').grade(b'fixture image', {'question': 'Find the area.'})
270
+ assert result['verdict'] == 'correct' and result['score'] == 100
271
+ assert len(requests) == 1
272
+ assert json.loads(requests[0]['messages'][0]['content'][0]['text']) == {
273
+ 'question_snapshot': {'question': 'Find the area.'}}
274
+ finally:
275
+ client.close()