evalarc / coding /index.html
glayguo's picture
Publish verified EvalArc 9d5d871ede42
05e3169 verified
Raw History Blame Contribute Delete
29.9 kB
<!doctype html>
<html lang="en"><meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta http-equiv="Content-Security-Policy" content="default-src 'none'; style-src 'unsafe-inline'; script-src 'sha256-KSx2iQ0M1S6PGsAFdxjeO1s+Kz8N9L6DA4i+wBqtNlY='; base-uri 'none'; form-action 'none'">
<meta name="color-scheme" content="light dark">
<title>EvalArc &#183; Grader audit</title>
<style>
:root{color-scheme:light dark;--bg:#101618;--surface:#152019;--surface-2:#17221c;
--text:#e0ece7;--muted:#acbeb5;--border:#33463d;--accent:#96e8b9;--accent-2:#86dbaf;
--bad:#edb68d;--warn:#e6d48a;--pass-bg:#152019;--fail-bg:#211b16;--warn-bg:#1f1d14;
--radius:12px;font:16px/1.6 system-ui,-apple-system,"Segoe UI",sans-serif;
background:var(--bg);color:var(--text)}
@media(prefers-color-scheme:light){:root{--bg:#ffffff;--surface:#f3f7f5;--surface-2:#eef3f0;
--text:#142019;--muted:#46564e;--border:#c9d6cf;--accent:#0b6b43;--accent-2:#0b6b43;
--bad:#a3420c;--warn:#7a5d00;--pass-bg:#eef8f2;--fail-bg:#fff4ec;--warn-bg:#fffbe6}}
*,*::before,*::after{box-sizing:border-box}
body{max-width:1120px;margin:40px auto;padding:0 24px}
.eyebrow{color:var(--accent-2);letter-spacing:.14em;font-size:12px;font-weight:700}
h1{font-size:clamp(28px,4vw,44px);letter-spacing:-.03em;line-height:1.15;margin:10px 0 18px}
h2{margin-top:40px;font-size:22px}h3{font-size:17px}
p{color:var(--muted);max-width:76ch}.cards{display:flex;gap:16px;flex-wrap:wrap;margin:24px 0}
.card{border:1px solid var(--border);border-radius:var(--radius);padding:16px 22px;flex:1;
min-width:150px;background:var(--surface)}
.number{font-size:32px;color:var(--accent);font-weight:650;font-variant-numeric:tabular-nums}
.label{color:var(--muted)}
.scroll{overflow:auto;max-height:70vh;border:1px solid var(--border);border-radius:var(--radius)}
table{border-collapse:collapse;width:100%;font-size:14px;font-variant-numeric:tabular-nums}
td,th{text-align:left;padding:10px 14px;border-bottom:1px solid var(--border);vertical-align:top}
th{color:var(--accent-2);background:var(--surface);position:sticky;top:0;z-index:1}
tbody tr:hover{background:var(--surface-2)}tbody tr:last-child td{border-bottom:0}
code{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:.92em}
.dimension{display:flex;gap:16px;align-items:center;max-width:620px;margin:10px 0}
.dimension span:first-child{width:150px}meter{flex:1;accent-color:var(--accent)}
footer{margin-top:48px;border-top:1px solid var(--border);padding-top:20px;font-size:13px;
color:var(--muted)}
.metadata{overflow-wrap:anywhere;font-size:13px}
pre{overflow:auto;padding:16px;background:var(--surface-2);font-size:12px;max-height:480px;
border-radius:8px;white-space:pre-wrap}
summary{cursor:pointer;color:var(--accent)}details{margin:12px 0}
summary{min-height:44px;padding:10px 0}
a:focus-visible,summary:focus-visible,button:focus-visible,input:focus-visible,
.scroll:focus-visible{outline:3px solid var(--accent);outline-offset:3px}
.audit-control{border:1px solid var(--border);border-radius:var(--radius);padding:12px 18px}
.audit-control summary{overflow-wrap:anywhere}.audit-control p{overflow-wrap:anywhere}
.audit-coverage{border-left:3px solid var(--bad);padding:12px 20px;background:var(--surface-2)}
a{color:var(--accent)}.failed,.agent_error,.environment_error{color:var(--bad)}
.passed{color:var(--accent)}td{overflow-wrap:anywhere}
.scroll table{min-width:680px}td:first-child{min-width:150px;overflow-wrap:normal}
.change-cards{display:none}
.change-card{border:1px solid var(--border);border-radius:var(--radius);padding:16px}
.change-card h3{font-size:16px;margin:0;overflow-wrap:anywhere}.change-card p{margin:10px 0 0}
.verdict{border:1px solid var(--border);border-left:6px solid var(--accent);
border-radius:var(--radius);padding:16px 22px;margin:20px 0 8px;background:var(--pass-bg)}
.verdict.fail{border-left-color:var(--bad);background:var(--fail-bg)}
.verdict.warn{border-left-color:var(--warn);background:var(--warn-bg)}
.verdict h2{margin:0;font-size:21px}.verdict p{margin:6px 0 0;color:var(--text)}
.tone{white-space:nowrap}.tone.ok{color:var(--accent)}.tone.bad{color:var(--bad)}
.tone.warn{color:var(--warn)}.tone.info{color:var(--muted)}
.findings{list-style:none;padding:0;display:grid;gap:10px;max-width:900px}
.findings li{border:1px solid var(--border);border-radius:10px;padding:12px 16px}
.skip{position:absolute;left:-200vw}
.skip:focus{left:24px;top:12px;background:var(--bg);padding:8px;z-index:5}
td code{overflow-wrap:anywhere}
.tools{display:flex;gap:12px;align-items:center;flex-wrap:wrap;margin:8px 0}
.tools input{font:inherit;padding:8px 12px;border:1px solid var(--border);border-radius:8px;
background:var(--bg);color:var(--text);min-height:44px;min-width:min(320px,100%)}
th button{all:unset;cursor:pointer;display:inline-flex;gap:6px;align-items:center;
min-height:32px;color:inherit;font-weight:700}
th button:focus-visible{outline:3px solid var(--accent);outline-offset:2px;border-radius:4px}
th button span{opacity:.6}[aria-sort] button span{opacity:1}
.count{font-size:13px;color:var(--muted)}
@media(max-width:640px){body{margin:24px auto;padding:0 16px}.cards{gap:12px}
.card{min-width:0;flex-basis:100%}.dimension{flex-wrap:wrap}.audit-control{padding:10px 14px}
.change-table{display:none}.change-cards{display:grid;gap:16px}
.scroll table{min-width:560px}td,th{padding:8px 10px}
main .cards .card{flex-basis:calc(50% - 6px);padding:12px 14px}main .number{font-size:24px}}
@media print{:root{--bg:#fff;--surface:#fff;--surface-2:#f4f4f4;--text:#111;--muted:#222;
--border:#bbb;--accent:#064;--accent-2:#064;--bad:#8a3b00;--warn:#6a5000;--pass-bg:#fff;
--fail-bg:#fff;--warn-bg:#fff}body{margin:0;max-width:none}
.verdict,.card,.findings li,tr{break-inside:avoid}pre{max-height:none}
.scroll{overflow:visible;max-height:none;border:0}th{position:static}
.skip,.tools{display:none}}
@media(prefers-reduced-motion:reduce){*{scroll-behavior:auto!important}}
</style>
<a class="skip" href="#main">Skip to content</a><main id="main">
<div class="eyebrow">EVALARC / DEVELOPMENT AUDIT</div>
<h1>Test the grader.<br>Then trust the signal.</h1>
<section class="verdict pass" role="status" aria-label="Result"><h2>&#10003; 8 of 8 declared faults detected</h2><p>3 fault(s) depend on a single detecting case.</p><p class="next"><strong>Next:</strong> Add cases that detect the single-case faults independently.</p></section><p>Behavioral controls for task-specific agent evaluations. Correct and faulty
submissions are evaluated against the same externally enforced contract.</p>
<div class="cards"><div class="card"><div class="number">8/8</div><div class="label">declared faults detected</div></div><div class="card"><div class="number">1.000</div><div class="label">reference correctness</div></div><div class="card"><div class="number">docker</div><div class="label">execution backend</div></div></div><section class="audit-coverage" aria-labelledby="coverage-title"><h2 id="coverage-title">Which faults depend on one case?</h2><p><strong>3 of 8</strong> declared faults have exactly one distinct detecting case in the assessed records. Repeating a case under more seeds does not increase this margin. A surviving fault has margin zero; an environment failure remains unassessed.</p><p>Margins below are derived from the saved failing case IDs. Rendering an older audit does not rerun its controls.</p></section><h2>Does the grader detect plausible defects?</h2><div class="scroll" tabindex="0" role="region" aria-label="Declared fault coverage"><table><thead><tr><th scope="col">Negative control</th><th scope="col">Target</th><th scope="col">Candidate score</th><th scope="col">Result</th><th scope="col" title="Cases that caught this fault independently. One means the suite loses this fault if that case changes.">Margin</th><th scope="col">Evidence</th></tr></thead><tbody><tr><td><code>ack-without-work</code></td><td>basic</td><td>0.000</td><td>Detected</td><td>3</td><td><a href="#fault-0">Inspect recorded cases</a></td></tr><tr><td><code>memory-only</code></td><td>persistence</td><td>0.783</td><td>Detected</td><td>3</td><td><a href="#fault-1">Inspect recorded cases</a></td></tr><tr><td><code>partial-batch</code></td><td>transactions</td><td>0.900</td><td>Detected</td><td>1</td><td><a href="#fault-2">Inspect recorded cases</a></td></tr><tr><td><code>cas-always-wins</code></td><td>compare_swap</td><td>0.850</td><td>Detected</td><td>2</td><td><a href="#fault-3">Inspect recorded cases</a></td></tr><tr><td><code>boolean-equals-one</code></td><td>compare_swap</td><td>0.925</td><td>Detected</td><td>1</td><td><a href="#fault-4">Inspect recorded cases</a></td></tr><tr><td><code>delete-noop</code></td><td>basic</td><td>0.650</td><td>Detected</td><td>2</td><td><a href="#fault-5">Inspect recorded cases</a></td></tr><tr><td><code>accept-nonstring-keys</code></td><td>validation</td><td>0.925</td><td>Detected</td><td>1</td><td><a href="#fault-6">Inspect recorded cases</a></td></tr><tr><td><code>commit-on-exit</code></td><td>crash_recovery</td><td>0.900</td><td>Detected</td><td>3</td><td><a href="#fault-7">Inspect recorded cases</a></td></tr></tbody></table></div><h2>Inspect the detecting cases</h2><details class="audit-control"><summary>ack-without-work &#183; 3 detecting cases</summary><div id="fault-0"><p>Target: basic. Detecting case IDs: json-types-and-overwrite, read-write-delete, seeded-state-machine.</p><details><summary>read-write-delete &#183; seed 17 &#183; failed</summary><pre>{
&quot;seed&quot;: 17,
&quot;case_id&quot;: &quot;read-write-delete&quot;,
&quot;dimension&quot;: &quot;basic&quot;,
&quot;checks&quot;: {
&quot;basic&quot;: false
},
&quot;passed&quot;: false,
&quot;status&quot;: &quot;failed&quot;,
&quot;checks_completed&quot;: 0,
&quot;duration_seconds&quot;: 0.608378,
&quot;transcript_sha256&quot;: &quot;e9d1c1a4ce952f925f5a3d00a84348bf482007670ba13602f73d0129cc422a60&quot;,
&quot;error&quot;: &quot;response mismatch at request 1; expected {\&quot;found\&quot;: false, \&quot;ok\&quot;: true}, got {\&quot;ok\&quot;: true}&quot;
}</pre></details><details><summary>json-types-and-overwrite &#183; seed 17 &#183; failed</summary><pre>{
&quot;seed&quot;: 17,
&quot;case_id&quot;: &quot;json-types-and-overwrite&quot;,
&quot;dimension&quot;: &quot;basic&quot;,
&quot;checks&quot;: {
&quot;basic&quot;: false
},
&quot;passed&quot;: false,
&quot;status&quot;: &quot;failed&quot;,
&quot;checks_completed&quot;: 1,
&quot;duration_seconds&quot;: 0.563549,
&quot;transcript_sha256&quot;: &quot;ad684a2e4e62254b35236e279cda7ec03bfe6bf6a01e3c0b24a1810e6c8088d2&quot;,
&quot;error&quot;: &quot;response mismatch at request 2; expected {\&quot;found\&quot;: true, \&quot;ok\&quot;: true, \&quot;value\&quot;: null}, got {\&quot;ok\&quot;: true}&quot;
}</pre></details><details><summary>seeded-state-machine &#183; seed 17 &#183; failed</summary><pre>{
&quot;seed&quot;: 17,
&quot;case_id&quot;: &quot;seeded-state-machine&quot;,
&quot;dimension&quot;: &quot;basic&quot;,
&quot;checks&quot;: {
&quot;basic&quot;: false
},
&quot;passed&quot;: false,
&quot;status&quot;: &quot;failed&quot;,
&quot;checks_completed&quot;: 2,
&quot;duration_seconds&quot;: 0.497166,
&quot;transcript_sha256&quot;: &quot;ce6e264c70b36102aabe44145d2a7558766913a2725261b8f120efea98fa8659&quot;,
&quot;error&quot;: &quot;response mismatch at request 3; expected {\&quot;found\&quot;: false, \&quot;ok\&quot;: true}, got {\&quot;ok\&quot;: true}&quot;
}</pre></details></div></details><details class="audit-control"><summary>memory-only &#183; 3 detecting cases</summary><div id="fault-1"><p>Target: persistence. Detecting case IDs: restart-preserves-acknowledged-write, restart-preserves-cas, restart-preserves-transaction.</p><details><summary>restart-preserves-acknowledged-write &#183; seed 17 &#183; failed</summary><pre>{
&quot;seed&quot;: 17,
&quot;case_id&quot;: &quot;restart-preserves-acknowledged-write&quot;,
&quot;dimension&quot;: &quot;persistence&quot;,
&quot;checks&quot;: {
&quot;persistence&quot;: false
},
&quot;passed&quot;: false,
&quot;status&quot;: &quot;failed&quot;,
&quot;checks_completed&quot;: 1,
&quot;duration_seconds&quot;: 0.976095,
&quot;transcript_sha256&quot;: &quot;f62cc3fb4db094e24ad23141f7cc50908d24647156c4d2ba2386a2487c078e80&quot;,
&quot;error&quot;: &quot;response mismatch at request 2; expected {\&quot;found\&quot;: true, \&quot;ok\&quot;: true, \&quot;value\&quot;: {\&quot;items\&quot;: [true, null, \&quot;\\u03bb\\n\\u4e16\\u754c\&quot;], \&quot;version\&quot;: 5991}}, got {\&quot;found\&quot;: false, \&quot;ok\&quot;: true}&quot;
}</pre></details><details><summary>restart-preserves-transaction &#183; seed 17 &#183; failed</summary><pre>{
&quot;seed&quot;: 17,
&quot;case_id&quot;: &quot;restart-preserves-transaction&quot;,
&quot;dimension&quot;: &quot;persistence&quot;,
&quot;checks&quot;: {
&quot;persistence&quot;: false
},
&quot;passed&quot;: false,
&quot;status&quot;: &quot;failed&quot;,
&quot;checks_completed&quot;: 1,
&quot;duration_seconds&quot;: 0.996261,
&quot;transcript_sha256&quot;: &quot;43cf3772fd4bd95486b1824b1d73c4e99859fb85088a7f62192d8325b4581b29&quot;,
&quot;error&quot;: &quot;response mismatch at request 2; expected {\&quot;found\&quot;: true, \&quot;ok\&quot;: true, \&quot;value\&quot;: {\&quot;items\&quot;: [true, null, \&quot;\\u03bb\\n\\u4e16\\u754c\&quot;], \&quot;version\&quot;: 5991}}, got {\&quot;found\&quot;: false, \&quot;ok\&quot;: true}&quot;
}</pre></details><details><summary>restart-preserves-cas &#183; seed 17 &#183; failed</summary><pre>{
&quot;seed&quot;: 17,
&quot;case_id&quot;: &quot;restart-preserves-cas&quot;,
&quot;dimension&quot;: &quot;persistence&quot;,
&quot;checks&quot;: {
&quot;persistence&quot;: false
},
&quot;passed&quot;: false,
&quot;status&quot;: &quot;failed&quot;,
&quot;checks_completed&quot;: 2,
&quot;duration_seconds&quot;: 0.997537,
&quot;transcript_sha256&quot;: &quot;497082839111501c6aa4912312fe30686befcab05beb84ebbcb99be0d1e8aef1&quot;,
&quot;error&quot;: &quot;response mismatch at request 3; expected {\&quot;found\&quot;: true, \&quot;ok\&quot;: true, \&quot;value\&quot;: 321}, got {\&quot;found\&quot;: false, \&quot;ok\&quot;: true}&quot;
}</pre></details></div></details><details class="audit-control"><summary>partial-batch &#183; single-case dependency</summary><div id="fault-2"><p>Target: transactions. Detecting case IDs: rollback-batch.</p><details><summary>rollback-batch &#183; seed 17 &#183; failed</summary><pre>{
&quot;seed&quot;: 17,
&quot;case_id&quot;: &quot;rollback-batch&quot;,
&quot;dimension&quot;: &quot;transactions&quot;,
&quot;checks&quot;: {
&quot;transactions&quot;: false
},
&quot;passed&quot;: false,
&quot;status&quot;: &quot;failed&quot;,
&quot;checks_completed&quot;: 2,
&quot;duration_seconds&quot;: 0.497347,
&quot;transcript_sha256&quot;: &quot;f6015865dce884224cf6ccccc9fbdd367baad17842dde00fbee476f06b1aab0d&quot;,
&quot;error&quot;: &quot;response mismatch at request 3; expected {\&quot;found\&quot;: true, \&quot;ok\&quot;: true, \&quot;value\&quot;: {\&quot;items\&quot;: [true, null, \&quot;\\u03bb\\n\\u4e16\\u754c\&quot;], \&quot;version\&quot;: 5991}}, got {\&quot;found\&quot;: true, \&quot;ok\&quot;: true, \&quot;value\&quot;: \&quot;must roll back\&quot;}&quot;
}</pre></details></div></details><details class="audit-control"><summary>cas-always-wins &#183; 2 detecting cases</summary><div id="fault-3"><p>Target: compare_swap. Detecting case IDs: cas-match-and-mismatch, cas-type-sensitivity.</p><details><summary>cas-match-and-mismatch &#183; seed 17 &#183; failed</summary><pre>{
&quot;seed&quot;: 17,
&quot;case_id&quot;: &quot;cas-match-and-mismatch&quot;,
&quot;dimension&quot;: &quot;compare_swap&quot;,
&quot;checks&quot;: {
&quot;compare_swap&quot;: false
},
&quot;passed&quot;: false,
&quot;status&quot;: &quot;failed&quot;,
&quot;checks_completed&quot;: 1,
&quot;duration_seconds&quot;: 0.557099,
&quot;transcript_sha256&quot;: &quot;71199e89272547e9c1a1961a85a650b93c30e2f848912b6be28ea262907eda56&quot;,
&quot;error&quot;: &quot;response mismatch at request 2; expected {\&quot;ok\&quot;: true, \&quot;swapped\&quot;: false}, got {\&quot;ok\&quot;: true, \&quot;swapped\&quot;: true}&quot;
}</pre></details><details><summary>cas-type-sensitivity &#183; seed 17 &#183; failed</summary><pre>{
&quot;seed&quot;: 17,
&quot;case_id&quot;: &quot;cas-type-sensitivity&quot;,
&quot;dimension&quot;: &quot;compare_swap&quot;,
&quot;checks&quot;: {
&quot;compare_swap&quot;: false
},
&quot;passed&quot;: false,
&quot;status&quot;: &quot;failed&quot;,
&quot;checks_completed&quot;: 1,
&quot;duration_seconds&quot;: 0.565552,
&quot;transcript_sha256&quot;: &quot;70231a539612a6973a06489411646982ea651ca401223316f7f670e6560a18bf&quot;,
&quot;error&quot;: &quot;response mismatch at request 2; expected {\&quot;ok\&quot;: true, \&quot;swapped\&quot;: false}, got {\&quot;ok\&quot;: true, \&quot;swapped\&quot;: true}&quot;
}</pre></details></div></details><details class="audit-control"><summary>boolean-equals-one &#183; single-case dependency</summary><div id="fault-4"><p>Target: compare_swap. Detecting case IDs: cas-type-sensitivity.</p><details><summary>cas-type-sensitivity &#183; seed 17 &#183; failed</summary><pre>{
&quot;seed&quot;: 17,
&quot;case_id&quot;: &quot;cas-type-sensitivity&quot;,
&quot;dimension&quot;: &quot;compare_swap&quot;,
&quot;checks&quot;: {
&quot;compare_swap&quot;: false
},
&quot;passed&quot;: false,
&quot;status&quot;: &quot;failed&quot;,
&quot;checks_completed&quot;: 1,
&quot;duration_seconds&quot;: 0.57604,
&quot;transcript_sha256&quot;: &quot;70231a539612a6973a06489411646982ea651ca401223316f7f670e6560a18bf&quot;,
&quot;error&quot;: &quot;response mismatch at request 2; expected {\&quot;ok\&quot;: true, \&quot;swapped\&quot;: false}, got {\&quot;ok\&quot;: true, \&quot;swapped\&quot;: true}&quot;
}</pre></details></div></details><details class="audit-control"><summary>delete-noop &#183; 2 detecting cases</summary><div id="fault-5"><p>Target: basic. Detecting case IDs: read-write-delete, seeded-state-machine.</p><details><summary>read-write-delete &#183; seed 17 &#183; failed</summary><pre>{
&quot;seed&quot;: 17,
&quot;case_id&quot;: &quot;read-write-delete&quot;,
&quot;dimension&quot;: &quot;basic&quot;,
&quot;checks&quot;: {
&quot;basic&quot;: false
},
&quot;passed&quot;: false,
&quot;status&quot;: &quot;failed&quot;,
&quot;checks_completed&quot;: 4,
&quot;duration_seconds&quot;: 0.557506,
&quot;transcript_sha256&quot;: &quot;4122b55bc205e4703a3a9f62a5397de448a833f424e54edc914bbbf5465c70d1&quot;,
&quot;error&quot;: &quot;response mismatch at request 5; expected {\&quot;found\&quot;: false, \&quot;ok\&quot;: true}, got {\&quot;found\&quot;: true, \&quot;ok\&quot;: true, \&quot;value\&quot;: {\&quot;items\&quot;: [true, null, \&quot;\\u03bb\\n\\u4e16\\u754c\&quot;], \&quot;version\&quot;: 5991}}&quot;
}</pre></details><details><summary>seeded-state-machine &#183; seed 17 &#183; failed</summary><pre>{
&quot;seed&quot;: 17,
&quot;case_id&quot;: &quot;seeded-state-machine&quot;,
&quot;dimension&quot;: &quot;basic&quot;,
&quot;checks&quot;: {
&quot;basic&quot;: false
},
&quot;passed&quot;: false,
&quot;status&quot;: &quot;failed&quot;,
&quot;checks_completed&quot;: 8,
&quot;duration_seconds&quot;: 0.556806,
&quot;transcript_sha256&quot;: &quot;0132892f9b4bac6264eb6ef2dae9da8b7bac7a6a158b56357f1319c853615de7&quot;,
&quot;error&quot;: &quot;response mismatch at request 9; expected {\&quot;found\&quot;: false, \&quot;ok\&quot;: true}, got {\&quot;found\&quot;: true, \&quot;ok\&quot;: true, \&quot;value\&quot;: {\&quot;items\&quot;: [true, null, \&quot;\\u03bb\\n\\u4e16\\u754c\&quot;], \&quot;version\&quot;: 5991}}&quot;
}</pre></details></div></details><details class="audit-control"><summary>accept-nonstring-keys &#183; single-case dependency</summary><div id="fault-6"><p>Target: validation. Detecting case IDs: reject-and-continue.</p><details><summary>reject-and-continue &#183; seed 17 &#183; failed</summary><pre>{
&quot;seed&quot;: 17,
&quot;case_id&quot;: &quot;reject-and-continue&quot;,
&quot;dimension&quot;: &quot;validation&quot;,
&quot;checks&quot;: {
&quot;validation&quot;: false
},
&quot;passed&quot;: false,
&quot;status&quot;: &quot;failed&quot;,
&quot;checks_completed&quot;: 4,
&quot;duration_seconds&quot;: 0.519073,
&quot;transcript_sha256&quot;: &quot;dc07c86f871afd6b2f208e9751bdd459e52368d54d008ca0622a8edd61a7008b&quot;,
&quot;error&quot;: &quot;response mismatch at request 5; expected {\&quot;error\&quot;: \&quot;invalid_request\&quot;, \&quot;ok\&quot;: false}, got {\&quot;ok\&quot;: true}&quot;
}</pre></details></div></details><details class="audit-control"><summary>commit-on-exit &#183; 3 detecting cases</summary><div id="fault-7"><p>Target: crash_recovery. Detecting case IDs: sigkill-after-acknowledgement, sigkill-after-atomic-batch, sigkill-after-delete.</p><details><summary>sigkill-after-acknowledgement &#183; seed 17 &#183; failed</summary><pre>{
&quot;seed&quot;: 17,
&quot;case_id&quot;: &quot;sigkill-after-acknowledgement&quot;,
&quot;dimension&quot;: &quot;crash_recovery&quot;,
&quot;checks&quot;: {
&quot;crash_recovery&quot;: false
},
&quot;passed&quot;: false,
&quot;status&quot;: &quot;failed&quot;,
&quot;checks_completed&quot;: 1,
&quot;duration_seconds&quot;: 0.998141,
&quot;transcript_sha256&quot;: &quot;f62cc3fb4db094e24ad23141f7cc50908d24647156c4d2ba2386a2487c078e80&quot;,
&quot;error&quot;: &quot;response mismatch at request 2; expected {\&quot;found\&quot;: true, \&quot;ok\&quot;: true, \&quot;value\&quot;: {\&quot;items\&quot;: [true, null, \&quot;\\u03bb\\n\\u4e16\\u754c\&quot;], \&quot;version\&quot;: 5991}}, got {\&quot;found\&quot;: false, \&quot;ok\&quot;: true}&quot;
}</pre></details><details><summary>sigkill-after-atomic-batch &#183; seed 17 &#183; failed</summary><pre>{
&quot;seed&quot;: 17,
&quot;case_id&quot;: &quot;sigkill-after-atomic-batch&quot;,
&quot;dimension&quot;: &quot;crash_recovery&quot;,
&quot;checks&quot;: {
&quot;crash_recovery&quot;: false
},
&quot;passed&quot;: false,
&quot;status&quot;: &quot;failed&quot;,
&quot;checks_completed&quot;: 1,
&quot;duration_seconds&quot;: 0.978949,
&quot;transcript_sha256&quot;: &quot;43cf3772fd4bd95486b1824b1d73c4e99859fb85088a7f62192d8325b4581b29&quot;,
&quot;error&quot;: &quot;response mismatch at request 2; expected {\&quot;found\&quot;: true, \&quot;ok\&quot;: true, \&quot;value\&quot;: {\&quot;items\&quot;: [true, null, \&quot;\\u03bb\\n\\u4e16\\u754c\&quot;], \&quot;version\&quot;: 5991}}, got {\&quot;found\&quot;: false, \&quot;ok\&quot;: true}&quot;
}</pre></details><details><summary>sigkill-after-delete &#183; seed 17 &#183; failed</summary><pre>{
&quot;seed&quot;: 17,
&quot;case_id&quot;: &quot;sigkill-after-delete&quot;,
&quot;dimension&quot;: &quot;crash_recovery&quot;,
&quot;checks&quot;: {
&quot;crash_recovery&quot;: false
},
&quot;passed&quot;: false,
&quot;status&quot;: &quot;failed&quot;,
&quot;checks_completed&quot;: 2,
&quot;duration_seconds&quot;: 1.47289,
&quot;transcript_sha256&quot;: &quot;679e7ab1f7f833fb12551b29af4fdc1ad8e1051810e594a4da85fdc3420b8c42&quot;,
&quot;error&quot;: &quot;response mismatch at request 3; expected {\&quot;found\&quot;: false, \&quot;ok\&quot;: true}, got {\&quot;found\&quot;: true, \&quot;ok\&quot;: true, \&quot;value\&quot;: {\&quot;items\&quot;: [true, null, \&quot;\\u03bb\\n\\u4e16\\u754c\&quot;], \&quot;version\&quot;: 5991}}&quot;
}</pre></details></div></details><h2>Positive control</h2><div class="dimension"><span>basic</span><meter min="0" max="1" value="1.0">1.00</meter><span>3/3</span></div><div class="dimension"><span>validation</span><meter min="0" max="1" value="1.0">1.00</meter><span>2/2</span></div><div class="dimension"><span>transactions</span><meter min="0" max="1" value="1.0">1.00</meter><span>2/2</span></div><div class="dimension"><span>compare_swap</span><meter min="0" max="1" value="1.0">1.00</meter><span>2/2</span></div><div class="dimension"><span>persistence</span><meter min="0" max="1" value="1.0">1.00</meter><span>3/3</span></div><div class="dimension"><span>crash_recovery</span><meter min="0" max="1" value="1.0">1.00</meter><span>3/3</span></div><h2>Reproduction record</h2><p class="metadata">Task: durable-kv v0.1.0<br>Seeds: [17]<br>Grader SHA-256: 52caa3136434569f96097860bc9bf85eb7ada5f7f9890c0540db81afb441871d<br>Cases SHA-256: 842a16a1e65700cde92ee5f5aaaa10d494a9fc9d71bd2d99fb68147b19aad63a<br>Image ID: sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea<br>Created: 2026-09-14T01:29:51.125110+00:00</p><p><a href="audit.json">Complete JSON evidence for all controls</a></p><footer><strong>Scope:</strong> This is a scripted grader audit, not an AI leaderboard. Public seeds are not held-out evaluation data. Detection rates apply only to the listed fault models. No human time horizon, model quality, or RL improvement is inferred.</footer></main><script data-evalarc="enhance">"use strict";
// Progressive enhancement for EvalArc offline reports, compiled to
// src/evalarc/assets/report_enhance.js (committed, so `pip install` needs no Node).
// The Python core inlines the compiled file and allows it by CSP hash; without
// JavaScript every table still renders in full. Sortable columns follow the WAI-ARIA
// APG sortable-table pattern (a button inside each header, aria-sort on the sorted
// column only). Tables with 8+ rows get a labelled row filter with a live count.
// Tables marked data-static are left alone.
(() => {
const SORTABLE_MIN_ROWS = 3;
const FILTER_MIN_ROWS = 8;
function leadingNumber(text) {
const match = text.replace(/,/g, "").match(/[-+]?\d*\.?\d+/);
return match ? parseFloat(match[0]) : NaN;
}
function cellText(row, column) {
const cell = row.cells[column];
return cell ? (cell.textContent ?? "").trim() : "";
}
function compare(a, b) {
const na = leadingNumber(a);
const nb = leadingNumber(b);
if (!Number.isNaN(na) && !Number.isNaN(nb))
return na - nb;
return a.localeCompare(b, undefined, { numeric: true });
}
function makeSortable(body, headers) {
headers.forEach((th, column) => {
const label = (th.textContent ?? "").trim();
if (!label)
return;
const button = document.createElement("button");
button.type = "button";
const icon = document.createElement("span");
icon.setAttribute("aria-hidden", "true");
icon.textContent = "&#8597;";
button.append(document.createTextNode(`${label} `), icon);
th.textContent = "";
th.append(button);
button.addEventListener("click", () => {
const direction = th.getAttribute("aria-sort") === "ascending" ? "descending" : "ascending";
for (const other of headers) {
other.removeAttribute("aria-sort");
const mark = other.querySelector("button span");
if (mark)
mark.textContent = "&#8597;";
}
th.setAttribute("aria-sort", direction);
icon.textContent = direction === "ascending" ? "&#9650;" : "&#9660;";
const sorted = Array.from(body.rows).sort((x, y) => {
const order = compare(cellText(x, column), cellText(y, column));
return direction === "ascending" ? order : -order;
});
for (const row of sorted)
body.append(row);
});
});
}
function addFilter(table, rows, index) {
const id = `filter-${index}`;
const tools = document.createElement("div");
tools.className = "tools";
const label = document.createElement("label");
label.htmlFor = id;
label.textContent = "Filter rows";
const input = document.createElement("input");
input.type = "search";
input.id = id;
input.autocomplete = "off";
const count = document.createElement("span");
count.className = "count";
count.setAttribute("aria-live", "polite");
const update = () => {
const query = input.value.trim().toLowerCase();
let shown = 0;
for (const row of rows) {
const hit = !query || (row.textContent ?? "").toLowerCase().includes(query);
row.hidden = !hit;
if (hit)
shown += 1;
}
count.textContent = `${shown} of ${rows.length} rows`;
};
input.addEventListener("input", update);
tools.append(label, input, count);
(table.closest(".scroll") ?? table).before(tools);
update();
}
function enhance() {
let filters = 0;
document.querySelectorAll("main table").forEach((table) => {
const body = table.tBodies[0];
const head = table.tHead;
if (!body || !head || !head.rows[0])
return;
if (body.rows.length < SORTABLE_MIN_ROWS || table.dataset.static !== undefined)
return;
const rows = Array.from(body.rows);
makeSortable(body, Array.from(head.rows[0].cells));
if (rows.length >= FILTER_MIN_ROWS) {
filters += 1;
addFilter(table, rows, filters);
}
});
}
enhance();
})();
</script></html>