Spaces:
Running
Running
| { | |
| "schema_version": "evalarc.audit.v2", | |
| "valid": true, | |
| "reference_passed": true, | |
| "reference": { | |
| "schema_version": "evalarc.evaluation.v2", | |
| "created_at": "2026-09-14T01:29:51.125110+00:00", | |
| "evalarc_version": "0.2.0", | |
| "task": { | |
| "id": "durable-kv", | |
| "version": "0.1.0", | |
| "domain": "coding", | |
| "split": "public-development" | |
| }, | |
| "candidate_sha256": "4240da0ab584ebbda01a737b07bb351ee465fe494595b96f67fd3371daf0f3f2", | |
| "grader_sha256": "52caa3136434569f96097860bc9bf85eb7ada5f7f9890c0540db81afb441871d", | |
| "cases_sha256": "842a16a1e65700cde92ee5f5aaaa10d494a9fc9d71bd2d99fb68147b19aad63a", | |
| "runtime": { | |
| "backend": "docker", | |
| "image": "python:3.12-slim", | |
| "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea", | |
| "command": [ | |
| "{python}", | |
| "-I", | |
| "-B", | |
| "main.py", | |
| "{state}/store.db" | |
| ], | |
| "response_timeout_seconds": 10.0, | |
| "session_output_limit_bytes": 1048576, | |
| "python": "3.12.3", | |
| "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39" | |
| }, | |
| "seeds": [ | |
| 17 | |
| ], | |
| "valid": true, | |
| "status": "passed", | |
| "score": 1.0, | |
| "resolved": true, | |
| "dimensions": { | |
| "basic": { | |
| "passed": 3, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.25, | |
| "score": 1.0 | |
| }, | |
| "validation": { | |
| "passed": 2, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.15, | |
| "score": 1.0 | |
| }, | |
| "transactions": { | |
| "passed": 2, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.2, | |
| "score": 1.0 | |
| }, | |
| "compare_swap": { | |
| "passed": 2, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.15, | |
| "score": 1.0 | |
| }, | |
| "persistence": { | |
| "passed": 3, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.15, | |
| "score": 1.0 | |
| }, | |
| "crash_recovery": { | |
| "passed": 3, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.1, | |
| "score": 1.0 | |
| } | |
| }, | |
| "cases": [ | |
| { | |
| "seed": 17, | |
| "case_id": "read-write-delete", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 6, | |
| "duration_seconds": 0.926471, | |
| "transcript_sha256": "c809eed26dc56583f9385d7d4adca2d981d06a672dc1ccbc0ffc52aa228b86f7", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "json-types-and-overwrite", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 16, | |
| "duration_seconds": 0.992022, | |
| "transcript_sha256": "83f6377f4b57ac659e8162136107ca644158ee089c944892aa9817085e74bd60", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "reject-and-continue", | |
| "dimension": "validation", | |
| "checks": { | |
| "validation": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 10, | |
| "duration_seconds": 0.802672, | |
| "transcript_sha256": "d4e0c26837f54e0af94d4909e3cab21d86dedca8c8842d0daae9e7471198c2a0", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "empty-and-unicode-keys", | |
| "dimension": "validation", | |
| "checks": { | |
| "validation": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 4, | |
| "duration_seconds": 0.668748, | |
| "transcript_sha256": "e8abe374fc308ce0744e96d2920495b96bda3b026cec9b86fa055023e1e4e620", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "commit-batch", | |
| "dimension": "transactions", | |
| "checks": { | |
| "transactions": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 4, | |
| "duration_seconds": 0.654542, | |
| "transcript_sha256": "ac21f2f6bcc47e3c807fa4ed72300af0f5ef39809ba17f93423e37a0e177e718", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "rollback-batch", | |
| "dimension": "transactions", | |
| "checks": { | |
| "transactions": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 7, | |
| "duration_seconds": 0.69832, | |
| "transcript_sha256": "1786c093ba29dcad562531a44cdbe5fddbed9d3f4632c1c6ddfe736baa240f35", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "cas-match-and-mismatch", | |
| "dimension": "compare_swap", | |
| "checks": { | |
| "compare_swap": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 7, | |
| "duration_seconds": 0.662942, | |
| "transcript_sha256": "ecd6148f6f387b4ef5351b877afd4cf27ea2a04c4253a34220fd78de5442d89f", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "cas-type-sensitivity", | |
| "dimension": "compare_swap", | |
| "checks": { | |
| "compare_swap": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 5, | |
| "duration_seconds": 0.749806, | |
| "transcript_sha256": "459a5b74588217d8abcf8e138eb559364a1d01d1f23e8c43ca0127ba28240258", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-acknowledged-write", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 4, | |
| "duration_seconds": 2.199417, | |
| "transcript_sha256": "c97b82c67f26a1aa2fdc8ac57f50690dbbe13554d094059ffe6690a592df3a55", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-transaction", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 1.166844, | |
| "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-acknowledgement", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 2, | |
| "duration_seconds": 1.224396, | |
| "transcript_sha256": "544c6be916371ed5a0e901394c5e293733ca84e0ad20079a952f9c4b2821a728", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-atomic-batch", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 1.151733, | |
| "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-cas", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 1.160644, | |
| "transcript_sha256": "149c04fd1366e77603e631349740ae38bdff40aa69f6eb4c738979eb13bf3f49", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-delete", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 1.718475, | |
| "transcript_sha256": "b3dba7f28e6629bb22f49b0439e0f74580ca3263dd98e16061ac9a8f680b5df7", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "seeded-state-machine", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 52, | |
| "duration_seconds": 0.815656, | |
| "transcript_sha256": "2a7e5ad4bedce67faf4d53f289c3a1a7304345a0167b287418fdf7dd411030f8", | |
| "error": null | |
| } | |
| ], | |
| "evaluation_seconds": 15.605099, | |
| "agent_cost_usd": null, | |
| "agent_tokens": null | |
| }, | |
| "mutants": [ | |
| { | |
| "name": "ack-without-work", | |
| "target_dimension": "basic", | |
| "killed": true, | |
| "valid": true, | |
| "score": 0.0, | |
| "failing_cases": [ | |
| "json-types-and-overwrite", | |
| "read-write-delete", | |
| "seeded-state-machine" | |
| ], | |
| "candidate_sha256": "e80e9d07c1b2a54e4e68a717c3ede2e71dbc871f7bc9c0174cffd9d0ef466f39", | |
| "evaluation": { | |
| "schema_version": "evalarc.evaluation.v2", | |
| "created_at": "2026-09-14T01:30:02.514728+00:00", | |
| "evalarc_version": "0.2.0", | |
| "task": { | |
| "id": "durable-kv", | |
| "version": "0.1.0", | |
| "domain": "coding", | |
| "split": "public-development" | |
| }, | |
| "candidate_sha256": "e80e9d07c1b2a54e4e68a717c3ede2e71dbc871f7bc9c0174cffd9d0ef466f39", | |
| "grader_sha256": "52caa3136434569f96097860bc9bf85eb7ada5f7f9890c0540db81afb441871d", | |
| "cases_sha256": "842a16a1e65700cde92ee5f5aaaa10d494a9fc9d71bd2d99fb68147b19aad63a", | |
| "runtime": { | |
| "backend": "docker", | |
| "image": "python:3.12-slim", | |
| "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea", | |
| "command": [ | |
| "{python}", | |
| "-I", | |
| "-B", | |
| "main.py", | |
| "{state}/store.db" | |
| ], | |
| "response_timeout_seconds": 10.0, | |
| "session_output_limit_bytes": 1048576, | |
| "python": "3.12.3", | |
| "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39" | |
| }, | |
| "seeds": [ | |
| 17 | |
| ], | |
| "valid": true, | |
| "status": "failed", | |
| "score": 0.0, | |
| "resolved": false, | |
| "dimensions": { | |
| "basic": { | |
| "passed": 0, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.25, | |
| "score": 0.0 | |
| }, | |
| "validation": { | |
| "passed": 0, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.15, | |
| "score": 0.0 | |
| }, | |
| "transactions": { | |
| "passed": 0, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.2, | |
| "score": 0.0 | |
| }, | |
| "compare_swap": { | |
| "passed": 0, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.15, | |
| "score": 0.0 | |
| }, | |
| "persistence": { | |
| "passed": 0, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.15, | |
| "score": 0.0 | |
| }, | |
| "crash_recovery": { | |
| "passed": 0, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.1, | |
| "score": 0.0 | |
| } | |
| }, | |
| "cases": [ | |
| { | |
| "seed": 17, | |
| "case_id": "read-write-delete", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 0, | |
| "duration_seconds": 0.608378, | |
| "transcript_sha256": "e9d1c1a4ce952f925f5a3d00a84348bf482007670ba13602f73d0129cc422a60", | |
| "error": "response mismatch at request 1; expected {\"found\": false, \"ok\": true}, got {\"ok\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "json-types-and-overwrite", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 1, | |
| "duration_seconds": 0.563549, | |
| "transcript_sha256": "ad684a2e4e62254b35236e279cda7ec03bfe6bf6a01e3c0b24a1810e6c8088d2", | |
| "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": null}, got {\"ok\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "reject-and-continue", | |
| "dimension": "validation", | |
| "checks": { | |
| "validation": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 1, | |
| "duration_seconds": 0.596346, | |
| "transcript_sha256": "932f3ec3f66ee636feddf02978271d1af20d352d113d51580182f4956132fcbc", | |
| "error": "response mismatch at request 2; expected {\"error\": \"invalid_request\", \"ok\": false}, got {\"ok\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "empty-and-unicode-keys", | |
| "dimension": "validation", | |
| "checks": { | |
| "validation": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 1, | |
| "duration_seconds": 0.576032, | |
| "transcript_sha256": "dd5c4b23bf9661f05363e4be2791dffa554d3bb7a6226d1be64e58adc97e7907", | |
| "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"ok\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "commit-batch", | |
| "dimension": "transactions", | |
| "checks": { | |
| "transactions": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 1, | |
| "duration_seconds": 0.635267, | |
| "transcript_sha256": "660d89f2f1ccd8d228c1871197f23f08289d1c1fe244918053aa9aca3a9ce576", | |
| "error": "response mismatch at request 2; expected {\"found\": false, \"ok\": true}, got {\"ok\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "rollback-batch", | |
| "dimension": "transactions", | |
| "checks": { | |
| "transactions": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 1, | |
| "duration_seconds": 0.5925, | |
| "transcript_sha256": "8d05dbc72469fa9ab22e4943d0e9ed13a8733f443dc95a84a306daa02b2031b4", | |
| "error": "response mismatch at request 2; expected {\"error\": \"invalid_request\", \"ok\": false}, got {\"ok\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "cas-match-and-mismatch", | |
| "dimension": "compare_swap", | |
| "checks": { | |
| "compare_swap": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 1, | |
| "duration_seconds": 0.558764, | |
| "transcript_sha256": "92c53f111ed9d1c600b448aaf9e5a529f32d2f085433ad700cd369bd71c55185", | |
| "error": "response mismatch at request 2; expected {\"ok\": true, \"swapped\": false}, got {\"ok\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "cas-type-sensitivity", | |
| "dimension": "compare_swap", | |
| "checks": { | |
| "compare_swap": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 1, | |
| "duration_seconds": 0.689461, | |
| "transcript_sha256": "88cbf896bd041b5c3a80f99fc01f19d0b473a1a39cdb4f59d621a8abbcaa35e4", | |
| "error": "response mismatch at request 2; expected {\"ok\": true, \"swapped\": false}, got {\"ok\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-acknowledged-write", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 1, | |
| "duration_seconds": 1.031052, | |
| "transcript_sha256": "d01e5e8ceee977002ba418a5bf134418cd85c9b36813ea2fadfa7908c141b718", | |
| "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"ok\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-transaction", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 1, | |
| "duration_seconds": 1.008206, | |
| "transcript_sha256": "5c70c6333e7c60969ecad930a4b839122a1582a89cdf257e323ffa6fbbf3036f", | |
| "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"ok\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-acknowledgement", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 1, | |
| "duration_seconds": 1.008229, | |
| "transcript_sha256": "d01e5e8ceee977002ba418a5bf134418cd85c9b36813ea2fadfa7908c141b718", | |
| "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"ok\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-atomic-batch", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 1, | |
| "duration_seconds": 1.010499, | |
| "transcript_sha256": "5c70c6333e7c60969ecad930a4b839122a1582a89cdf257e323ffa6fbbf3036f", | |
| "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"ok\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-cas", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 1, | |
| "duration_seconds": 0.506207, | |
| "transcript_sha256": "7ef0ed9705196a8514631dacec2c92eeb5e34cd9811d281f9816e348c50d1a17", | |
| "error": "response mismatch at request 2; expected {\"ok\": true, \"swapped\": true}, got {\"ok\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-delete", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 2, | |
| "duration_seconds": 1.497816, | |
| "transcript_sha256": "165fee513b6ad33052636d17d8a1f492e3226d1b5f19b246f01f1c20289f5525", | |
| "error": "response mismatch at request 3; expected {\"found\": false, \"ok\": true}, got {\"ok\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "seeded-state-machine", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 2, | |
| "duration_seconds": 0.497166, | |
| "transcript_sha256": "ce6e264c70b36102aabe44145d2a7558766913a2725261b8f120efea98fa8659", | |
| "error": "response mismatch at request 3; expected {\"found\": false, \"ok\": true}, got {\"ok\": true}" | |
| } | |
| ], | |
| "evaluation_seconds": 11.386876, | |
| "agent_cost_usd": null, | |
| "agent_tokens": null | |
| } | |
| }, | |
| { | |
| "name": "memory-only", | |
| "target_dimension": "persistence", | |
| "killed": true, | |
| "valid": true, | |
| "score": 0.78333333, | |
| "failing_cases": [ | |
| "restart-preserves-acknowledged-write", | |
| "restart-preserves-cas", | |
| "restart-preserves-transaction" | |
| ], | |
| "candidate_sha256": "478fc026e785f7dcad2dd83744abbba6c482dc1f903e6ece8b0803dd335389e1", | |
| "evaluation": { | |
| "schema_version": "evalarc.evaluation.v2", | |
| "created_at": "2026-09-14T01:30:13.553761+00:00", | |
| "evalarc_version": "0.2.0", | |
| "task": { | |
| "id": "durable-kv", | |
| "version": "0.1.0", | |
| "domain": "coding", | |
| "split": "public-development" | |
| }, | |
| "candidate_sha256": "478fc026e785f7dcad2dd83744abbba6c482dc1f903e6ece8b0803dd335389e1", | |
| "grader_sha256": "52caa3136434569f96097860bc9bf85eb7ada5f7f9890c0540db81afb441871d", | |
| "cases_sha256": "842a16a1e65700cde92ee5f5aaaa10d494a9fc9d71bd2d99fb68147b19aad63a", | |
| "runtime": { | |
| "backend": "docker", | |
| "image": "python:3.12-slim", | |
| "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea", | |
| "command": [ | |
| "{python}", | |
| "-I", | |
| "-B", | |
| "main.py", | |
| "{state}/store.db" | |
| ], | |
| "response_timeout_seconds": 10.0, | |
| "session_output_limit_bytes": 1048576, | |
| "python": "3.12.3", | |
| "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39" | |
| }, | |
| "seeds": [ | |
| 17 | |
| ], | |
| "valid": true, | |
| "status": "failed", | |
| "score": 0.78333333, | |
| "resolved": false, | |
| "dimensions": { | |
| "basic": { | |
| "passed": 3, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.25, | |
| "score": 1.0 | |
| }, | |
| "validation": { | |
| "passed": 2, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.15, | |
| "score": 1.0 | |
| }, | |
| "transactions": { | |
| "passed": 2, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.2, | |
| "score": 1.0 | |
| }, | |
| "compare_swap": { | |
| "passed": 2, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.15, | |
| "score": 1.0 | |
| }, | |
| "persistence": { | |
| "passed": 0, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.15, | |
| "score": 0.0 | |
| }, | |
| "crash_recovery": { | |
| "passed": 1, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.1, | |
| "score": 0.3333333333333333 | |
| } | |
| }, | |
| "cases": [ | |
| { | |
| "seed": 17, | |
| "case_id": "read-write-delete", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 6, | |
| "duration_seconds": 0.510333, | |
| "transcript_sha256": "c809eed26dc56583f9385d7d4adca2d981d06a672dc1ccbc0ffc52aa228b86f7", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "json-types-and-overwrite", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 16, | |
| "duration_seconds": 0.512203, | |
| "transcript_sha256": "83f6377f4b57ac659e8162136107ca644158ee089c944892aa9817085e74bd60", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "reject-and-continue", | |
| "dimension": "validation", | |
| "checks": { | |
| "validation": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 10, | |
| "duration_seconds": 0.5002, | |
| "transcript_sha256": "d4e0c26837f54e0af94d4909e3cab21d86dedca8c8842d0daae9e7471198c2a0", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "empty-and-unicode-keys", | |
| "dimension": "validation", | |
| "checks": { | |
| "validation": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 4, | |
| "duration_seconds": 0.499981, | |
| "transcript_sha256": "e8abe374fc308ce0744e96d2920495b96bda3b026cec9b86fa055023e1e4e620", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "commit-batch", | |
| "dimension": "transactions", | |
| "checks": { | |
| "transactions": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 4, | |
| "duration_seconds": 0.487121, | |
| "transcript_sha256": "ac21f2f6bcc47e3c807fa4ed72300af0f5ef39809ba17f93423e37a0e177e718", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "rollback-batch", | |
| "dimension": "transactions", | |
| "checks": { | |
| "transactions": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 7, | |
| "duration_seconds": 0.492795, | |
| "transcript_sha256": "1786c093ba29dcad562531a44cdbe5fddbed9d3f4632c1c6ddfe736baa240f35", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "cas-match-and-mismatch", | |
| "dimension": "compare_swap", | |
| "checks": { | |
| "compare_swap": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 7, | |
| "duration_seconds": 0.491807, | |
| "transcript_sha256": "ecd6148f6f387b4ef5351b877afd4cf27ea2a04c4253a34220fd78de5442d89f", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "cas-type-sensitivity", | |
| "dimension": "compare_swap", | |
| "checks": { | |
| "compare_swap": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 5, | |
| "duration_seconds": 0.494448, | |
| "transcript_sha256": "459a5b74588217d8abcf8e138eb559364a1d01d1f23e8c43ca0127ba28240258", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-acknowledged-write", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 1, | |
| "duration_seconds": 0.976095, | |
| "transcript_sha256": "f62cc3fb4db094e24ad23141f7cc50908d24647156c4d2ba2386a2487c078e80", | |
| "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"found\": false, \"ok\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-transaction", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 1, | |
| "duration_seconds": 0.996261, | |
| "transcript_sha256": "43cf3772fd4bd95486b1824b1d73c4e99859fb85088a7f62192d8325b4581b29", | |
| "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"found\": false, \"ok\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-acknowledgement", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 1, | |
| "duration_seconds": 1.054678, | |
| "transcript_sha256": "f62cc3fb4db094e24ad23141f7cc50908d24647156c4d2ba2386a2487c078e80", | |
| "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"found\": false, \"ok\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-atomic-batch", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 1, | |
| "duration_seconds": 1.013921, | |
| "transcript_sha256": "43cf3772fd4bd95486b1824b1d73c4e99859fb85088a7f62192d8325b4581b29", | |
| "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"found\": false, \"ok\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-cas", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 2, | |
| "duration_seconds": 0.997537, | |
| "transcript_sha256": "497082839111501c6aa4912312fe30686befcab05beb84ebbcb99be0d1e8aef1", | |
| "error": "response mismatch at request 3; expected {\"found\": true, \"ok\": true, \"value\": 321}, got {\"found\": false, \"ok\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-delete", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 1.494879, | |
| "transcript_sha256": "b3dba7f28e6629bb22f49b0439e0f74580ca3263dd98e16061ac9a8f680b5df7", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "seeded-state-machine", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 52, | |
| "duration_seconds": 0.509886, | |
| "transcript_sha256": "2a7e5ad4bedce67faf4d53f289c3a1a7304345a0167b287418fdf7dd411030f8", | |
| "error": null | |
| } | |
| ], | |
| "evaluation_seconds": 11.038733, | |
| "agent_cost_usd": null, | |
| "agent_tokens": null | |
| } | |
| }, | |
| { | |
| "name": "partial-batch", | |
| "target_dimension": "transactions", | |
| "killed": true, | |
| "valid": true, | |
| "score": 0.9, | |
| "failing_cases": [ | |
| "rollback-batch" | |
| ], | |
| "candidate_sha256": "833f852d0c053ed18f935d9daf03bb36a174f517c54fe75675f874b05036140a", | |
| "evaluation": { | |
| "schema_version": "evalarc.evaluation.v2", | |
| "created_at": "2026-09-14T01:30:25.787520+00:00", | |
| "evalarc_version": "0.2.0", | |
| "task": { | |
| "id": "durable-kv", | |
| "version": "0.1.0", | |
| "domain": "coding", | |
| "split": "public-development" | |
| }, | |
| "candidate_sha256": "833f852d0c053ed18f935d9daf03bb36a174f517c54fe75675f874b05036140a", | |
| "grader_sha256": "52caa3136434569f96097860bc9bf85eb7ada5f7f9890c0540db81afb441871d", | |
| "cases_sha256": "842a16a1e65700cde92ee5f5aaaa10d494a9fc9d71bd2d99fb68147b19aad63a", | |
| "runtime": { | |
| "backend": "docker", | |
| "image": "python:3.12-slim", | |
| "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea", | |
| "command": [ | |
| "{python}", | |
| "-I", | |
| "-B", | |
| "main.py", | |
| "{state}/store.db" | |
| ], | |
| "response_timeout_seconds": 10.0, | |
| "session_output_limit_bytes": 1048576, | |
| "python": "3.12.3", | |
| "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39" | |
| }, | |
| "seeds": [ | |
| 17 | |
| ], | |
| "valid": true, | |
| "status": "failed", | |
| "score": 0.9, | |
| "resolved": false, | |
| "dimensions": { | |
| "basic": { | |
| "passed": 3, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.25, | |
| "score": 1.0 | |
| }, | |
| "validation": { | |
| "passed": 2, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.15, | |
| "score": 1.0 | |
| }, | |
| "transactions": { | |
| "passed": 1, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.2, | |
| "score": 0.5 | |
| }, | |
| "compare_swap": { | |
| "passed": 2, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.15, | |
| "score": 1.0 | |
| }, | |
| "persistence": { | |
| "passed": 3, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.15, | |
| "score": 1.0 | |
| }, | |
| "crash_recovery": { | |
| "passed": 3, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.1, | |
| "score": 1.0 | |
| } | |
| }, | |
| "cases": [ | |
| { | |
| "seed": 17, | |
| "case_id": "read-write-delete", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 6, | |
| "duration_seconds": 0.531285, | |
| "transcript_sha256": "c809eed26dc56583f9385d7d4adca2d981d06a672dc1ccbc0ffc52aa228b86f7", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "json-types-and-overwrite", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 16, | |
| "duration_seconds": 0.542066, | |
| "transcript_sha256": "83f6377f4b57ac659e8162136107ca644158ee089c944892aa9817085e74bd60", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "reject-and-continue", | |
| "dimension": "validation", | |
| "checks": { | |
| "validation": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 10, | |
| "duration_seconds": 0.514736, | |
| "transcript_sha256": "d4e0c26837f54e0af94d4909e3cab21d86dedca8c8842d0daae9e7471198c2a0", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "empty-and-unicode-keys", | |
| "dimension": "validation", | |
| "checks": { | |
| "validation": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 4, | |
| "duration_seconds": 0.511955, | |
| "transcript_sha256": "e8abe374fc308ce0744e96d2920495b96bda3b026cec9b86fa055023e1e4e620", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "commit-batch", | |
| "dimension": "transactions", | |
| "checks": { | |
| "transactions": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 4, | |
| "duration_seconds": 0.506471, | |
| "transcript_sha256": "ac21f2f6bcc47e3c807fa4ed72300af0f5ef39809ba17f93423e37a0e177e718", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "rollback-batch", | |
| "dimension": "transactions", | |
| "checks": { | |
| "transactions": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 2, | |
| "duration_seconds": 0.497347, | |
| "transcript_sha256": "f6015865dce884224cf6ccccc9fbdd367baad17842dde00fbee476f06b1aab0d", | |
| "error": "response mismatch at request 3; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"found\": true, \"ok\": true, \"value\": \"must roll back\"}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "cas-match-and-mismatch", | |
| "dimension": "compare_swap", | |
| "checks": { | |
| "compare_swap": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 7, | |
| "duration_seconds": 0.533444, | |
| "transcript_sha256": "ecd6148f6f387b4ef5351b877afd4cf27ea2a04c4253a34220fd78de5442d89f", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "cas-type-sensitivity", | |
| "dimension": "compare_swap", | |
| "checks": { | |
| "compare_swap": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 5, | |
| "duration_seconds": 0.498659, | |
| "transcript_sha256": "459a5b74588217d8abcf8e138eb559364a1d01d1f23e8c43ca0127ba28240258", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-acknowledged-write", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 4, | |
| "duration_seconds": 1.499686, | |
| "transcript_sha256": "c97b82c67f26a1aa2fdc8ac57f50690dbbe13554d094059ffe6690a592df3a55", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-transaction", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 0.997884, | |
| "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-acknowledgement", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 2, | |
| "duration_seconds": 0.99713, | |
| "transcript_sha256": "544c6be916371ed5a0e901394c5e293733ca84e0ad20079a952f9c4b2821a728", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-atomic-batch", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 1.008529, | |
| "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-cas", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 1.060491, | |
| "transcript_sha256": "149c04fd1366e77603e631349740ae38bdff40aa69f6eb4c738979eb13bf3f49", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-delete", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 1.694327, | |
| "transcript_sha256": "b3dba7f28e6629bb22f49b0439e0f74580ca3263dd98e16061ac9a8f680b5df7", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "seeded-state-machine", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 52, | |
| "duration_seconds": 0.830685, | |
| "transcript_sha256": "2a7e5ad4bedce67faf4d53f289c3a1a7304345a0167b287418fdf7dd411030f8", | |
| "error": null | |
| } | |
| ], | |
| "evaluation_seconds": 12.23358, | |
| "agent_cost_usd": null, | |
| "agent_tokens": null | |
| } | |
| }, | |
| { | |
| "name": "cas-always-wins", | |
| "target_dimension": "compare_swap", | |
| "killed": true, | |
| "valid": true, | |
| "score": 0.85, | |
| "failing_cases": [ | |
| "cas-match-and-mismatch", | |
| "cas-type-sensitivity" | |
| ], | |
| "candidate_sha256": "eb503c0ba87888db36fea991fc389223855c3e5d4c75ad9979a92e6109846b80", | |
| "evaluation": { | |
| "schema_version": "evalarc.evaluation.v2", | |
| "created_at": "2026-09-14T01:30:39.354373+00:00", | |
| "evalarc_version": "0.2.0", | |
| "task": { | |
| "id": "durable-kv", | |
| "version": "0.1.0", | |
| "domain": "coding", | |
| "split": "public-development" | |
| }, | |
| "candidate_sha256": "eb503c0ba87888db36fea991fc389223855c3e5d4c75ad9979a92e6109846b80", | |
| "grader_sha256": "52caa3136434569f96097860bc9bf85eb7ada5f7f9890c0540db81afb441871d", | |
| "cases_sha256": "842a16a1e65700cde92ee5f5aaaa10d494a9fc9d71bd2d99fb68147b19aad63a", | |
| "runtime": { | |
| "backend": "docker", | |
| "image": "python:3.12-slim", | |
| "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea", | |
| "command": [ | |
| "{python}", | |
| "-I", | |
| "-B", | |
| "main.py", | |
| "{state}/store.db" | |
| ], | |
| "response_timeout_seconds": 10.0, | |
| "session_output_limit_bytes": 1048576, | |
| "python": "3.12.3", | |
| "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39" | |
| }, | |
| "seeds": [ | |
| 17 | |
| ], | |
| "valid": true, | |
| "status": "failed", | |
| "score": 0.85, | |
| "resolved": false, | |
| "dimensions": { | |
| "basic": { | |
| "passed": 3, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.25, | |
| "score": 1.0 | |
| }, | |
| "validation": { | |
| "passed": 2, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.15, | |
| "score": 1.0 | |
| }, | |
| "transactions": { | |
| "passed": 2, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.2, | |
| "score": 1.0 | |
| }, | |
| "compare_swap": { | |
| "passed": 0, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.15, | |
| "score": 0.0 | |
| }, | |
| "persistence": { | |
| "passed": 3, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.15, | |
| "score": 1.0 | |
| }, | |
| "crash_recovery": { | |
| "passed": 3, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.1, | |
| "score": 1.0 | |
| } | |
| }, | |
| "cases": [ | |
| { | |
| "seed": 17, | |
| "case_id": "read-write-delete", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 6, | |
| "duration_seconds": 0.601743, | |
| "transcript_sha256": "c809eed26dc56583f9385d7d4adca2d981d06a672dc1ccbc0ffc52aa228b86f7", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "json-types-and-overwrite", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 16, | |
| "duration_seconds": 0.666199, | |
| "transcript_sha256": "83f6377f4b57ac659e8162136107ca644158ee089c944892aa9817085e74bd60", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "reject-and-continue", | |
| "dimension": "validation", | |
| "checks": { | |
| "validation": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 10, | |
| "duration_seconds": 0.582939, | |
| "transcript_sha256": "d4e0c26837f54e0af94d4909e3cab21d86dedca8c8842d0daae9e7471198c2a0", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "empty-and-unicode-keys", | |
| "dimension": "validation", | |
| "checks": { | |
| "validation": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 4, | |
| "duration_seconds": 0.591822, | |
| "transcript_sha256": "e8abe374fc308ce0744e96d2920495b96bda3b026cec9b86fa055023e1e4e620", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "commit-batch", | |
| "dimension": "transactions", | |
| "checks": { | |
| "transactions": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 4, | |
| "duration_seconds": 0.550826, | |
| "transcript_sha256": "ac21f2f6bcc47e3c807fa4ed72300af0f5ef39809ba17f93423e37a0e177e718", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "rollback-batch", | |
| "dimension": "transactions", | |
| "checks": { | |
| "transactions": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 7, | |
| "duration_seconds": 0.558259, | |
| "transcript_sha256": "1786c093ba29dcad562531a44cdbe5fddbed9d3f4632c1c6ddfe736baa240f35", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "cas-match-and-mismatch", | |
| "dimension": "compare_swap", | |
| "checks": { | |
| "compare_swap": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 1, | |
| "duration_seconds": 0.557099, | |
| "transcript_sha256": "71199e89272547e9c1a1961a85a650b93c30e2f848912b6be28ea262907eda56", | |
| "error": "response mismatch at request 2; expected {\"ok\": true, \"swapped\": false}, got {\"ok\": true, \"swapped\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "cas-type-sensitivity", | |
| "dimension": "compare_swap", | |
| "checks": { | |
| "compare_swap": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 1, | |
| "duration_seconds": 0.565552, | |
| "transcript_sha256": "70231a539612a6973a06489411646982ea651ca401223316f7f670e6560a18bf", | |
| "error": "response mismatch at request 2; expected {\"ok\": true, \"swapped\": false}, got {\"ok\": true, \"swapped\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-acknowledged-write", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 4, | |
| "duration_seconds": 1.660789, | |
| "transcript_sha256": "c97b82c67f26a1aa2fdc8ac57f50690dbbe13554d094059ffe6690a592df3a55", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-transaction", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 1.16438, | |
| "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-acknowledgement", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 2, | |
| "duration_seconds": 1.396278, | |
| "transcript_sha256": "544c6be916371ed5a0e901394c5e293733ca84e0ad20079a952f9c4b2821a728", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-atomic-batch", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 1.131043, | |
| "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-cas", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 1.129017, | |
| "transcript_sha256": "149c04fd1366e77603e631349740ae38bdff40aa69f6eb4c738979eb13bf3f49", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-delete", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 1.646182, | |
| "transcript_sha256": "b3dba7f28e6629bb22f49b0439e0f74580ca3263dd98e16061ac9a8f680b5df7", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "seeded-state-machine", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 52, | |
| "duration_seconds": 0.753312, | |
| "transcript_sha256": "2a7e5ad4bedce67faf4d53f289c3a1a7304345a0167b287418fdf7dd411030f8", | |
| "error": null | |
| } | |
| ], | |
| "evaluation_seconds": 13.566574, | |
| "agent_cost_usd": null, | |
| "agent_tokens": null | |
| } | |
| }, | |
| { | |
| "name": "boolean-equals-one", | |
| "target_dimension": "compare_swap", | |
| "killed": true, | |
| "valid": true, | |
| "score": 0.925, | |
| "failing_cases": [ | |
| "cas-type-sensitivity" | |
| ], | |
| "candidate_sha256": "8b3b2b2e4d41f2b0cec36bdc4aef42dad3969f9d81b9a0a0d6cde665d538aac3", | |
| "evaluation": { | |
| "schema_version": "evalarc.evaluation.v2", | |
| "created_at": "2026-09-14T01:30:52.200917+00:00", | |
| "evalarc_version": "0.2.0", | |
| "task": { | |
| "id": "durable-kv", | |
| "version": "0.1.0", | |
| "domain": "coding", | |
| "split": "public-development" | |
| }, | |
| "candidate_sha256": "8b3b2b2e4d41f2b0cec36bdc4aef42dad3969f9d81b9a0a0d6cde665d538aac3", | |
| "grader_sha256": "52caa3136434569f96097860bc9bf85eb7ada5f7f9890c0540db81afb441871d", | |
| "cases_sha256": "842a16a1e65700cde92ee5f5aaaa10d494a9fc9d71bd2d99fb68147b19aad63a", | |
| "runtime": { | |
| "backend": "docker", | |
| "image": "python:3.12-slim", | |
| "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea", | |
| "command": [ | |
| "{python}", | |
| "-I", | |
| "-B", | |
| "main.py", | |
| "{state}/store.db" | |
| ], | |
| "response_timeout_seconds": 10.0, | |
| "session_output_limit_bytes": 1048576, | |
| "python": "3.12.3", | |
| "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39" | |
| }, | |
| "seeds": [ | |
| 17 | |
| ], | |
| "valid": true, | |
| "status": "failed", | |
| "score": 0.925, | |
| "resolved": false, | |
| "dimensions": { | |
| "basic": { | |
| "passed": 3, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.25, | |
| "score": 1.0 | |
| }, | |
| "validation": { | |
| "passed": 2, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.15, | |
| "score": 1.0 | |
| }, | |
| "transactions": { | |
| "passed": 2, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.2, | |
| "score": 1.0 | |
| }, | |
| "compare_swap": { | |
| "passed": 1, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.15, | |
| "score": 0.5 | |
| }, | |
| "persistence": { | |
| "passed": 3, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.15, | |
| "score": 1.0 | |
| }, | |
| "crash_recovery": { | |
| "passed": 3, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.1, | |
| "score": 1.0 | |
| } | |
| }, | |
| "cases": [ | |
| { | |
| "seed": 17, | |
| "case_id": "read-write-delete", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 6, | |
| "duration_seconds": 0.556055, | |
| "transcript_sha256": "c809eed26dc56583f9385d7d4adca2d981d06a672dc1ccbc0ffc52aa228b86f7", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "json-types-and-overwrite", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 16, | |
| "duration_seconds": 0.602162, | |
| "transcript_sha256": "83f6377f4b57ac659e8162136107ca644158ee089c944892aa9817085e74bd60", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "reject-and-continue", | |
| "dimension": "validation", | |
| "checks": { | |
| "validation": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 10, | |
| "duration_seconds": 0.538341, | |
| "transcript_sha256": "d4e0c26837f54e0af94d4909e3cab21d86dedca8c8842d0daae9e7471198c2a0", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "empty-and-unicode-keys", | |
| "dimension": "validation", | |
| "checks": { | |
| "validation": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 4, | |
| "duration_seconds": 0.550951, | |
| "transcript_sha256": "e8abe374fc308ce0744e96d2920495b96bda3b026cec9b86fa055023e1e4e620", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "commit-batch", | |
| "dimension": "transactions", | |
| "checks": { | |
| "transactions": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 4, | |
| "duration_seconds": 0.551653, | |
| "transcript_sha256": "ac21f2f6bcc47e3c807fa4ed72300af0f5ef39809ba17f93423e37a0e177e718", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "rollback-batch", | |
| "dimension": "transactions", | |
| "checks": { | |
| "transactions": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 7, | |
| "duration_seconds": 0.559257, | |
| "transcript_sha256": "1786c093ba29dcad562531a44cdbe5fddbed9d3f4632c1c6ddfe736baa240f35", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "cas-match-and-mismatch", | |
| "dimension": "compare_swap", | |
| "checks": { | |
| "compare_swap": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 7, | |
| "duration_seconds": 0.552929, | |
| "transcript_sha256": "ecd6148f6f387b4ef5351b877afd4cf27ea2a04c4253a34220fd78de5442d89f", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "cas-type-sensitivity", | |
| "dimension": "compare_swap", | |
| "checks": { | |
| "compare_swap": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 1, | |
| "duration_seconds": 0.57604, | |
| "transcript_sha256": "70231a539612a6973a06489411646982ea651ca401223316f7f670e6560a18bf", | |
| "error": "response mismatch at request 2; expected {\"ok\": true, \"swapped\": false}, got {\"ok\": true, \"swapped\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-acknowledged-write", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 4, | |
| "duration_seconds": 1.639025, | |
| "transcript_sha256": "c97b82c67f26a1aa2fdc8ac57f50690dbbe13554d094059ffe6690a592df3a55", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-transaction", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 1.062938, | |
| "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-acknowledgement", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 2, | |
| "duration_seconds": 1.049137, | |
| "transcript_sha256": "544c6be916371ed5a0e901394c5e293733ca84e0ad20079a952f9c4b2821a728", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-atomic-batch", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 1.085374, | |
| "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-cas", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 1.08237, | |
| "transcript_sha256": "149c04fd1366e77603e631349740ae38bdff40aa69f6eb4c738979eb13bf3f49", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-delete", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 1.6509, | |
| "transcript_sha256": "b3dba7f28e6629bb22f49b0439e0f74580ca3263dd98e16061ac9a8f680b5df7", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "seeded-state-machine", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 52, | |
| "duration_seconds": 0.780467, | |
| "transcript_sha256": "2a7e5ad4bedce67faf4d53f289c3a1a7304345a0167b287418fdf7dd411030f8", | |
| "error": null | |
| } | |
| ], | |
| "evaluation_seconds": 12.84625, | |
| "agent_cost_usd": null, | |
| "agent_tokens": null | |
| } | |
| }, | |
| { | |
| "name": "delete-noop", | |
| "target_dimension": "basic", | |
| "killed": true, | |
| "valid": true, | |
| "score": 0.65, | |
| "failing_cases": [ | |
| "read-write-delete", | |
| "seeded-state-machine" | |
| ], | |
| "candidate_sha256": "2c81d23c574991d7a80b45df1b79ada77309e930e73b8a41a8d0a1e4440625f8", | |
| "evaluation": { | |
| "schema_version": "evalarc.evaluation.v2", | |
| "created_at": "2026-09-14T01:31:04.674524+00:00", | |
| "evalarc_version": "0.2.0", | |
| "task": { | |
| "id": "durable-kv", | |
| "version": "0.1.0", | |
| "domain": "coding", | |
| "split": "public-development" | |
| }, | |
| "candidate_sha256": "2c81d23c574991d7a80b45df1b79ada77309e930e73b8a41a8d0a1e4440625f8", | |
| "grader_sha256": "52caa3136434569f96097860bc9bf85eb7ada5f7f9890c0540db81afb441871d", | |
| "cases_sha256": "842a16a1e65700cde92ee5f5aaaa10d494a9fc9d71bd2d99fb68147b19aad63a", | |
| "runtime": { | |
| "backend": "docker", | |
| "image": "python:3.12-slim", | |
| "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea", | |
| "command": [ | |
| "{python}", | |
| "-I", | |
| "-B", | |
| "main.py", | |
| "{state}/store.db" | |
| ], | |
| "response_timeout_seconds": 10.0, | |
| "session_output_limit_bytes": 1048576, | |
| "python": "3.12.3", | |
| "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39" | |
| }, | |
| "seeds": [ | |
| 17 | |
| ], | |
| "valid": true, | |
| "status": "failed", | |
| "score": 0.65, | |
| "resolved": false, | |
| "dimensions": { | |
| "basic": { | |
| "passed": 1, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.25, | |
| "score": 0.3333333333333333 | |
| }, | |
| "validation": { | |
| "passed": 2, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.15, | |
| "score": 1.0 | |
| }, | |
| "transactions": { | |
| "passed": 1, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.2, | |
| "score": 0.5 | |
| }, | |
| "compare_swap": { | |
| "passed": 2, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.15, | |
| "score": 1.0 | |
| }, | |
| "persistence": { | |
| "passed": 2, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.15, | |
| "score": 0.6666666666666666 | |
| }, | |
| "crash_recovery": { | |
| "passed": 2, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.1, | |
| "score": 0.6666666666666666 | |
| } | |
| }, | |
| "cases": [ | |
| { | |
| "seed": 17, | |
| "case_id": "read-write-delete", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 4, | |
| "duration_seconds": 0.557506, | |
| "transcript_sha256": "4122b55bc205e4703a3a9f62a5397de448a833f424e54edc914bbbf5465c70d1", | |
| "error": "response mismatch at request 5; expected {\"found\": false, \"ok\": true}, got {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "json-types-and-overwrite", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 16, | |
| "duration_seconds": 0.623245, | |
| "transcript_sha256": "83f6377f4b57ac659e8162136107ca644158ee089c944892aa9817085e74bd60", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "reject-and-continue", | |
| "dimension": "validation", | |
| "checks": { | |
| "validation": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 10, | |
| "duration_seconds": 0.55319, | |
| "transcript_sha256": "d4e0c26837f54e0af94d4909e3cab21d86dedca8c8842d0daae9e7471198c2a0", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "empty-and-unicode-keys", | |
| "dimension": "validation", | |
| "checks": { | |
| "validation": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 4, | |
| "duration_seconds": 0.527258, | |
| "transcript_sha256": "e8abe374fc308ce0744e96d2920495b96bda3b026cec9b86fa055023e1e4e620", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "commit-batch", | |
| "dimension": "transactions", | |
| "checks": { | |
| "transactions": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 1, | |
| "duration_seconds": 0.533816, | |
| "transcript_sha256": "da7df3388998e4f6fd4743177baa1ddcd8625d54fded8d5c6cfbdf813d50cdba", | |
| "error": "response mismatch at request 2; expected {\"found\": false, \"ok\": true}, got {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "rollback-batch", | |
| "dimension": "transactions", | |
| "checks": { | |
| "transactions": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 7, | |
| "duration_seconds": 0.571187, | |
| "transcript_sha256": "1786c093ba29dcad562531a44cdbe5fddbed9d3f4632c1c6ddfe736baa240f35", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "cas-match-and-mismatch", | |
| "dimension": "compare_swap", | |
| "checks": { | |
| "compare_swap": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 7, | |
| "duration_seconds": 0.53584, | |
| "transcript_sha256": "ecd6148f6f387b4ef5351b877afd4cf27ea2a04c4253a34220fd78de5442d89f", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "cas-type-sensitivity", | |
| "dimension": "compare_swap", | |
| "checks": { | |
| "compare_swap": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 5, | |
| "duration_seconds": 0.568142, | |
| "transcript_sha256": "459a5b74588217d8abcf8e138eb559364a1d01d1f23e8c43ca0127ba28240258", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-acknowledged-write", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 3, | |
| "duration_seconds": 1.643947, | |
| "transcript_sha256": "0ae3e20082ef3346a8027b09f61e373e1c733234603e3e12d5e61853d6df9c8f", | |
| "error": "response mismatch at request 4; expected {\"found\": false, \"ok\": true}, got {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-transaction", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 1.090287, | |
| "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-acknowledgement", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 2, | |
| "duration_seconds": 1.113412, | |
| "transcript_sha256": "544c6be916371ed5a0e901394c5e293733ca84e0ad20079a952f9c4b2821a728", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-atomic-batch", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 1.077148, | |
| "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-cas", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 1.085685, | |
| "transcript_sha256": "149c04fd1366e77603e631349740ae38bdff40aa69f6eb4c738979eb13bf3f49", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-delete", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 2, | |
| "duration_seconds": 1.429231, | |
| "transcript_sha256": "679e7ab1f7f833fb12551b29af4fdc1ad8e1051810e594a4da85fdc3420b8c42", | |
| "error": "response mismatch at request 3; expected {\"found\": false, \"ok\": true}, got {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "seeded-state-machine", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 8, | |
| "duration_seconds": 0.556806, | |
| "transcript_sha256": "0132892f9b4bac6264eb6ef2dae9da8b7bac7a6a158b56357f1319c853615de7", | |
| "error": "response mismatch at request 9; expected {\"found\": false, \"ok\": true}, got {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}" | |
| } | |
| ], | |
| "evaluation_seconds": 12.473283, | |
| "agent_cost_usd": null, | |
| "agent_tokens": null | |
| } | |
| }, | |
| { | |
| "name": "accept-nonstring-keys", | |
| "target_dimension": "validation", | |
| "killed": true, | |
| "valid": true, | |
| "score": 0.925, | |
| "failing_cases": [ | |
| "reject-and-continue" | |
| ], | |
| "candidate_sha256": "1c3846e123f39e86cf3302b5548674c429e4c740e178563ad9f9457ab784f5a4", | |
| "evaluation": { | |
| "schema_version": "evalarc.evaluation.v2", | |
| "created_at": "2026-09-14T01:31:16.266965+00:00", | |
| "evalarc_version": "0.2.0", | |
| "task": { | |
| "id": "durable-kv", | |
| "version": "0.1.0", | |
| "domain": "coding", | |
| "split": "public-development" | |
| }, | |
| "candidate_sha256": "1c3846e123f39e86cf3302b5548674c429e4c740e178563ad9f9457ab784f5a4", | |
| "grader_sha256": "52caa3136434569f96097860bc9bf85eb7ada5f7f9890c0540db81afb441871d", | |
| "cases_sha256": "842a16a1e65700cde92ee5f5aaaa10d494a9fc9d71bd2d99fb68147b19aad63a", | |
| "runtime": { | |
| "backend": "docker", | |
| "image": "python:3.12-slim", | |
| "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea", | |
| "command": [ | |
| "{python}", | |
| "-I", | |
| "-B", | |
| "main.py", | |
| "{state}/store.db" | |
| ], | |
| "response_timeout_seconds": 10.0, | |
| "session_output_limit_bytes": 1048576, | |
| "python": "3.12.3", | |
| "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39" | |
| }, | |
| "seeds": [ | |
| 17 | |
| ], | |
| "valid": true, | |
| "status": "failed", | |
| "score": 0.925, | |
| "resolved": false, | |
| "dimensions": { | |
| "basic": { | |
| "passed": 3, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.25, | |
| "score": 1.0 | |
| }, | |
| "validation": { | |
| "passed": 1, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.15, | |
| "score": 0.5 | |
| }, | |
| "transactions": { | |
| "passed": 2, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.2, | |
| "score": 1.0 | |
| }, | |
| "compare_swap": { | |
| "passed": 2, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.15, | |
| "score": 1.0 | |
| }, | |
| "persistence": { | |
| "passed": 3, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.15, | |
| "score": 1.0 | |
| }, | |
| "crash_recovery": { | |
| "passed": 3, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.1, | |
| "score": 1.0 | |
| } | |
| }, | |
| "cases": [ | |
| { | |
| "seed": 17, | |
| "case_id": "read-write-delete", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 6, | |
| "duration_seconds": 0.487851, | |
| "transcript_sha256": "c809eed26dc56583f9385d7d4adca2d981d06a672dc1ccbc0ffc52aa228b86f7", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "json-types-and-overwrite", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 16, | |
| "duration_seconds": 0.54511, | |
| "transcript_sha256": "83f6377f4b57ac659e8162136107ca644158ee089c944892aa9817085e74bd60", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "reject-and-continue", | |
| "dimension": "validation", | |
| "checks": { | |
| "validation": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 4, | |
| "duration_seconds": 0.519073, | |
| "transcript_sha256": "dc07c86f871afd6b2f208e9751bdd459e52368d54d008ca0622a8edd61a7008b", | |
| "error": "response mismatch at request 5; expected {\"error\": \"invalid_request\", \"ok\": false}, got {\"ok\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "empty-and-unicode-keys", | |
| "dimension": "validation", | |
| "checks": { | |
| "validation": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 4, | |
| "duration_seconds": 0.485221, | |
| "transcript_sha256": "e8abe374fc308ce0744e96d2920495b96bda3b026cec9b86fa055023e1e4e620", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "commit-batch", | |
| "dimension": "transactions", | |
| "checks": { | |
| "transactions": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 4, | |
| "duration_seconds": 0.509709, | |
| "transcript_sha256": "ac21f2f6bcc47e3c807fa4ed72300af0f5ef39809ba17f93423e37a0e177e718", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "rollback-batch", | |
| "dimension": "transactions", | |
| "checks": { | |
| "transactions": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 7, | |
| "duration_seconds": 0.509348, | |
| "transcript_sha256": "1786c093ba29dcad562531a44cdbe5fddbed9d3f4632c1c6ddfe736baa240f35", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "cas-match-and-mismatch", | |
| "dimension": "compare_swap", | |
| "checks": { | |
| "compare_swap": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 7, | |
| "duration_seconds": 0.515143, | |
| "transcript_sha256": "ecd6148f6f387b4ef5351b877afd4cf27ea2a04c4253a34220fd78de5442d89f", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "cas-type-sensitivity", | |
| "dimension": "compare_swap", | |
| "checks": { | |
| "compare_swap": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 5, | |
| "duration_seconds": 0.501223, | |
| "transcript_sha256": "459a5b74588217d8abcf8e138eb559364a1d01d1f23e8c43ca0127ba28240258", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-acknowledged-write", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 4, | |
| "duration_seconds": 1.484043, | |
| "transcript_sha256": "c97b82c67f26a1aa2fdc8ac57f50690dbbe13554d094059ffe6690a592df3a55", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-transaction", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 0.98699, | |
| "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-acknowledgement", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 2, | |
| "duration_seconds": 0.981747, | |
| "transcript_sha256": "544c6be916371ed5a0e901394c5e293733ca84e0ad20079a952f9c4b2821a728", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-atomic-batch", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 0.987597, | |
| "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-cas", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 0.981076, | |
| "transcript_sha256": "149c04fd1366e77603e631349740ae38bdff40aa69f6eb4c738979eb13bf3f49", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-delete", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 1.437775, | |
| "transcript_sha256": "b3dba7f28e6629bb22f49b0439e0f74580ca3263dd98e16061ac9a8f680b5df7", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "seeded-state-machine", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 52, | |
| "duration_seconds": 0.653102, | |
| "transcript_sha256": "2a7e5ad4bedce67faf4d53f289c3a1a7304345a0167b287418fdf7dd411030f8", | |
| "error": null | |
| } | |
| ], | |
| "evaluation_seconds": 11.592179, | |
| "agent_cost_usd": null, | |
| "agent_tokens": null | |
| } | |
| }, | |
| { | |
| "name": "commit-on-exit", | |
| "target_dimension": "crash_recovery", | |
| "killed": true, | |
| "valid": true, | |
| "score": 0.9, | |
| "failing_cases": [ | |
| "sigkill-after-acknowledgement", | |
| "sigkill-after-atomic-batch", | |
| "sigkill-after-delete" | |
| ], | |
| "candidate_sha256": "fce3eb722763136ecc184b4cc5fd02f80623615ed44a438caf69f561e4cc438f", | |
| "evaluation": { | |
| "schema_version": "evalarc.evaluation.v2", | |
| "created_at": "2026-09-14T01:31:27.622867+00:00", | |
| "evalarc_version": "0.2.0", | |
| "task": { | |
| "id": "durable-kv", | |
| "version": "0.1.0", | |
| "domain": "coding", | |
| "split": "public-development" | |
| }, | |
| "candidate_sha256": "fce3eb722763136ecc184b4cc5fd02f80623615ed44a438caf69f561e4cc438f", | |
| "grader_sha256": "52caa3136434569f96097860bc9bf85eb7ada5f7f9890c0540db81afb441871d", | |
| "cases_sha256": "842a16a1e65700cde92ee5f5aaaa10d494a9fc9d71bd2d99fb68147b19aad63a", | |
| "runtime": { | |
| "backend": "docker", | |
| "image": "python:3.12-slim", | |
| "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea", | |
| "command": [ | |
| "{python}", | |
| "-I", | |
| "-B", | |
| "main.py", | |
| "{state}/store.db" | |
| ], | |
| "response_timeout_seconds": 10.0, | |
| "session_output_limit_bytes": 1048576, | |
| "python": "3.12.3", | |
| "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39" | |
| }, | |
| "seeds": [ | |
| 17 | |
| ], | |
| "valid": true, | |
| "status": "failed", | |
| "score": 0.9, | |
| "resolved": false, | |
| "dimensions": { | |
| "basic": { | |
| "passed": 3, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.25, | |
| "score": 1.0 | |
| }, | |
| "validation": { | |
| "passed": 2, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.15, | |
| "score": 1.0 | |
| }, | |
| "transactions": { | |
| "passed": 2, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.2, | |
| "score": 1.0 | |
| }, | |
| "compare_swap": { | |
| "passed": 2, | |
| "total": 2, | |
| "assessed": 2, | |
| "weight": 0.15, | |
| "score": 1.0 | |
| }, | |
| "persistence": { | |
| "passed": 3, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.15, | |
| "score": 1.0 | |
| }, | |
| "crash_recovery": { | |
| "passed": 0, | |
| "total": 3, | |
| "assessed": 3, | |
| "weight": 0.1, | |
| "score": 0.0 | |
| } | |
| }, | |
| "cases": [ | |
| { | |
| "seed": 17, | |
| "case_id": "read-write-delete", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 6, | |
| "duration_seconds": 0.498981, | |
| "transcript_sha256": "c809eed26dc56583f9385d7d4adca2d981d06a672dc1ccbc0ffc52aa228b86f7", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "json-types-and-overwrite", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 16, | |
| "duration_seconds": 0.510524, | |
| "transcript_sha256": "83f6377f4b57ac659e8162136107ca644158ee089c944892aa9817085e74bd60", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "reject-and-continue", | |
| "dimension": "validation", | |
| "checks": { | |
| "validation": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 10, | |
| "duration_seconds": 0.488127, | |
| "transcript_sha256": "d4e0c26837f54e0af94d4909e3cab21d86dedca8c8842d0daae9e7471198c2a0", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "empty-and-unicode-keys", | |
| "dimension": "validation", | |
| "checks": { | |
| "validation": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 4, | |
| "duration_seconds": 0.489482, | |
| "transcript_sha256": "e8abe374fc308ce0744e96d2920495b96bda3b026cec9b86fa055023e1e4e620", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "commit-batch", | |
| "dimension": "transactions", | |
| "checks": { | |
| "transactions": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 4, | |
| "duration_seconds": 0.499994, | |
| "transcript_sha256": "ac21f2f6bcc47e3c807fa4ed72300af0f5ef39809ba17f93423e37a0e177e718", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "rollback-batch", | |
| "dimension": "transactions", | |
| "checks": { | |
| "transactions": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 7, | |
| "duration_seconds": 0.533228, | |
| "transcript_sha256": "1786c093ba29dcad562531a44cdbe5fddbed9d3f4632c1c6ddfe736baa240f35", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "cas-match-and-mismatch", | |
| "dimension": "compare_swap", | |
| "checks": { | |
| "compare_swap": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 7, | |
| "duration_seconds": 0.502809, | |
| "transcript_sha256": "ecd6148f6f387b4ef5351b877afd4cf27ea2a04c4253a34220fd78de5442d89f", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "cas-type-sensitivity", | |
| "dimension": "compare_swap", | |
| "checks": { | |
| "compare_swap": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 5, | |
| "duration_seconds": 0.496033, | |
| "transcript_sha256": "459a5b74588217d8abcf8e138eb559364a1d01d1f23e8c43ca0127ba28240258", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-acknowledged-write", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 4, | |
| "duration_seconds": 1.433264, | |
| "transcript_sha256": "c97b82c67f26a1aa2fdc8ac57f50690dbbe13554d094059ffe6690a592df3a55", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-transaction", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 0.995327, | |
| "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-acknowledgement", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 1, | |
| "duration_seconds": 0.998141, | |
| "transcript_sha256": "f62cc3fb4db094e24ad23141f7cc50908d24647156c4d2ba2386a2487c078e80", | |
| "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"found\": false, \"ok\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-atomic-batch", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 1, | |
| "duration_seconds": 0.978949, | |
| "transcript_sha256": "43cf3772fd4bd95486b1824b1d73c4e99859fb85088a7f62192d8325b4581b29", | |
| "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"found\": false, \"ok\": true}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "restart-preserves-cas", | |
| "dimension": "persistence", | |
| "checks": { | |
| "persistence": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 3, | |
| "duration_seconds": 0.939053, | |
| "transcript_sha256": "149c04fd1366e77603e631349740ae38bdff40aa69f6eb4c738979eb13bf3f49", | |
| "error": null | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "sigkill-after-delete", | |
| "dimension": "crash_recovery", | |
| "checks": { | |
| "crash_recovery": false | |
| }, | |
| "passed": false, | |
| "status": "failed", | |
| "checks_completed": 2, | |
| "duration_seconds": 1.47289, | |
| "transcript_sha256": "679e7ab1f7f833fb12551b29af4fdc1ad8e1051810e594a4da85fdc3420b8c42", | |
| "error": "response mismatch at request 3; expected {\"found\": false, \"ok\": true}, got {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}" | |
| }, | |
| { | |
| "seed": 17, | |
| "case_id": "seeded-state-machine", | |
| "dimension": "basic", | |
| "checks": { | |
| "basic": true | |
| }, | |
| "passed": true, | |
| "status": "passed", | |
| "checks_completed": 52, | |
| "duration_seconds": 0.511465, | |
| "transcript_sha256": "2a7e5ad4bedce67faf4d53f289c3a1a7304345a0167b287418fdf7dd411030f8", | |
| "error": null | |
| } | |
| ], | |
| "evaluation_seconds": 11.355773, | |
| "agent_cost_usd": null, | |
| "agent_tokens": null | |
| } | |
| } | |
| ], | |
| "mutation_score": 1.0, | |
| "killed": 8, | |
| "total": 8, | |
| "passed": true, | |
| "interpretation": "Coverage of 8 declared behavioral fault models only. This is not a bound on reward hacking or a model benchmark." | |
| } | |