File size: 4,199 Bytes
2d82128
 
 
 
c816150
2d82128
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
c816150
2d82128
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
/**
 * Check the Randomize button's output: valid against the schema, plausible
 * against clinical bands, and accepted by the real /predict endpoint.
 *
 *   node scripts/verify_randomize.mjs [baseUrl]
 *
 * Randomize used to sample uniformly across each field's SCHEMA range, which
 * is a validation bound and not a clinical distribution -- a quarter of
 * randomised patients had a resting BP over 180. There is no JS test runner
 * in this project, so this script is the repeatable check.
 *
 * Exit code 0 only if every assertion holds.
 */

import { parseSchema } from '../frontend/src/utils/schemaFieldParser.js';
import { randomPatient, profileFor } from '../frontend/src/utils/randomPatient.js';

const BASE = process.argv[2] ?? 'http://127.0.0.1:8000';
const N = 500;

let failures = 0;
function check(ok, message) {
  if (!ok) { console.log(`  FAIL  ${message}`); failures++; }
}

for (const disease of ['heart_disease', 'diabetes_nhanes']) {
  console.log(`\n${disease}`);
  const schema = await (await fetch(`${BASE}/api/v4/${disease}/schema`)).json();
  const fields = parseSchema(schema);

  const stats = {};
  const patients = [];
  for (let i = 0; i < N; i++) {
    const p = randomPatient(fields, disease);
    patients.push(p);
    for (const f of fields) {
      const v = p[f.name];
      const { minimum, maximum, enum: enumValues } = f.validation || {};

      // 1. valid against the schema
      if (enumValues?.length && f.component === 'select') {
        check(enumValues.includes(v), `${f.name}: ${v} is not one of ${enumValues}`);
      } else if (typeof v === 'number') {
        if (minimum !== undefined) check(v >= minimum, `${f.name}: ${v} < minimum ${minimum}`);
        if (maximum !== undefined) check(v <= maximum, `${f.name}: ${v} > maximum ${maximum}`);
        check(Number.isFinite(v), `${f.name}: ${v} is not finite`);
        stats[f.name] ??= { min: Infinity, max: -Infinity, sum: 0, n: 0 };
        const s = stats[f.name];
        s.min = Math.min(s.min, v); s.max = Math.max(s.max, v); s.sum += v; s.n++;
      }
    }
  }

  // 2. plausible against the declared clinical band
  for (const [name, s] of Object.entries(stats)) {
    const profile = profileFor(disease, name);
    const mean = s.sum / s.n;
    if (profile) {
      check(s.min >= profile.min - 1e-9, `${name}: sampled ${s.min} below band min ${profile.min}`);
      check(s.max <= profile.max + 1e-9, `${name}: sampled ${s.max} above band max ${profile.max}`);
      check(mean > profile.min && mean < profile.max, `${name}: mean ${mean} outside band`);
    }
    console.log(`  ${name.padEnd(22)} min=${s.min.toFixed(1).padStart(6)}  mean=${mean.toFixed(1).padStart(6)}  max=${s.max.toFixed(1).padStart(6)}${profile ? `   band ${profile.min}-${profile.max}` : ''}`);
  }

  // 3. the server accepts them -- a 422 here means Randomize produces input
  //    the product itself rejects, which is what a judge would hit.
  // /predict is rate limited (30/min per IP), and a 429 says nothing about
  // whether the patient was valid -- only a 422 does. They are counted
  // separately so a throttled run is not reported as a validation failure.
  let accepted = 0, invalid = 0, throttled = 0;
  for (const p of patients.slice(0, 12)) {
    const res = await fetch(`${BASE}/api/v4/${disease}/predict`, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify(p),
    });
    if (res.ok) { accepted++; continue; }
    if (res.status === 429) { throttled++; continue; }
    invalid++;
    if (invalid === 1) console.log(`  FIRST REJECTION (${res.status}): ${(await res.text()).slice(0, 300)}`);
  }
  check(invalid === 0, `${invalid} randomised patients were rejected as invalid by /predict`);
  check(accepted > 0, 'every /predict call was throttled — nothing was actually verified');
  console.log(`  /predict: ${accepted} accepted, ${invalid} invalid, ${throttled} throttled`);
  // Stay under the per-minute limit before the next disease.
  await new Promise((r) => setTimeout(r, 3000));
}

console.log(failures === 0 ? '\nPASS — all assertions held' : `\nFAIL — ${failures} assertion(s)`);
process.exit(failures === 0 ? 0 : 1);