omnidiag / scripts /verify_randomize.mjs
yahyoha's picture
Claude Opus 5.5
Release: retire BRFSS diabetes (source deploy/v2-platform 8bbbfe9)
c816150
Raw History Blame Contribute Delete
4.2 kB
/**
* Check the Randomize button's output: valid against the schema, plausible
* against clinical bands, and accepted by the real /predict endpoint.
*
* node scripts/verify_randomize.mjs [baseUrl]
*
* Randomize used to sample uniformly across each field's SCHEMA range, which
* is a validation bound and not a clinical distribution -- a quarter of
* randomised patients had a resting BP over 180. There is no JS test runner
* in this project, so this script is the repeatable check.
*
* Exit code 0 only if every assertion holds.
*/
import { parseSchema } from '../frontend/src/utils/schemaFieldParser.js';
import { randomPatient, profileFor } from '../frontend/src/utils/randomPatient.js';
const BASE = process.argv[2] ?? 'http://127.0.0.1:8000';
const N = 500;
let failures = 0;
function check(ok, message) {
if (!ok) { console.log(` FAIL ${message}`); failures++; }
}
for (const disease of ['heart_disease', 'diabetes_nhanes']) {
console.log(`\n${disease}`);
const schema = await (await fetch(`${BASE}/api/v4/${disease}/schema`)).json();
const fields = parseSchema(schema);
const stats = {};
const patients = [];
for (let i = 0; i < N; i++) {
const p = randomPatient(fields, disease);
patients.push(p);
for (const f of fields) {
const v = p[f.name];
const { minimum, maximum, enum: enumValues } = f.validation || {};
// 1. valid against the schema
if (enumValues?.length && f.component === 'select') {
check(enumValues.includes(v), `${f.name}: ${v} is not one of ${enumValues}`);
} else if (typeof v === 'number') {
if (minimum !== undefined) check(v >= minimum, `${f.name}: ${v} < minimum ${minimum}`);
if (maximum !== undefined) check(v <= maximum, `${f.name}: ${v} > maximum ${maximum}`);
check(Number.isFinite(v), `${f.name}: ${v} is not finite`);
stats[f.name] ??= { min: Infinity, max: -Infinity, sum: 0, n: 0 };
const s = stats[f.name];
s.min = Math.min(s.min, v); s.max = Math.max(s.max, v); s.sum += v; s.n++;
}
}
}
// 2. plausible against the declared clinical band
for (const [name, s] of Object.entries(stats)) {
const profile = profileFor(disease, name);
const mean = s.sum / s.n;
if (profile) {
check(s.min >= profile.min - 1e-9, `${name}: sampled ${s.min} below band min ${profile.min}`);
check(s.max <= profile.max + 1e-9, `${name}: sampled ${s.max} above band max ${profile.max}`);
check(mean > profile.min && mean < profile.max, `${name}: mean ${mean} outside band`);
}
console.log(` ${name.padEnd(22)} min=${s.min.toFixed(1).padStart(6)} mean=${mean.toFixed(1).padStart(6)} max=${s.max.toFixed(1).padStart(6)}${profile ? ` band ${profile.min}-${profile.max}` : ''}`);
}
// 3. the server accepts them -- a 422 here means Randomize produces input
// the product itself rejects, which is what a judge would hit.
// /predict is rate limited (30/min per IP), and a 429 says nothing about
// whether the patient was valid -- only a 422 does. They are counted
// separately so a throttled run is not reported as a validation failure.
let accepted = 0, invalid = 0, throttled = 0;
for (const p of patients.slice(0, 12)) {
const res = await fetch(`${BASE}/api/v4/${disease}/predict`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(p),
});
if (res.ok) { accepted++; continue; }
if (res.status === 429) { throttled++; continue; }
invalid++;
if (invalid === 1) console.log(` FIRST REJECTION (${res.status}): ${(await res.text()).slice(0, 300)}`);
}
check(invalid === 0, `${invalid} randomised patients were rejected as invalid by /predict`);
check(accepted > 0, 'every /predict call was throttled — nothing was actually verified');
console.log(` /predict: ${accepted} accepted, ${invalid} invalid, ${throttled} throttled`);
// Stay under the per-minute limit before the next disease.
await new Promise((r) => setTimeout(r, 3000));
}
console.log(failures === 0 ? '\nPASS — all assertions held' : `\nFAIL — ${failures} assertion(s)`);
process.exit(failures === 0 ? 0 : 1);