Blame for test_engine.js

const assert=require('node:assert/strict'),fs=require('fs');
const R=require('./robustness_engine'),M=require('./source_modelops_engine');
const data=JSON.parse(fs.readFileSync('robustness_snapshots.json','utf8')).rows;
const train=JSON.parse(fs.readFileSync('train.json','utf8'));
const iid=JSON.parse(fs.readFileSync('iid_test.json','utf8'));
const challenge=JSON.parse(fs.readFileSync('challenge.json','utf8'));
let pass=0;
const test=(title,fn)=>{try{fn();pass++;process.stdout.write('PASS '+title+'\n');}catch(err){console.error('FAIL '+title,err);process.exitCode=1;}};
function close(a,b,eps=1e-9){assert.ok(Math.abs(a-b)<eps,`${a} != ${b}`)}
test('original 842 training examples preserved',()=>assert.equal(train.length,842));
test('original 130 controlled examples preserved',()=>assert.equal(iid.length,130));
test('original 54 challenge examples preserved',()=>assert.equal(challenge.length,54));
test('four methods registered',()=>assert.deepEqual(R.METHODS,['nb','word','char','hybrid']));
test('six test conditions registered',()=>assert.equal(R.ATTACKS.length,6));
test('truncate explicitly noninvariant',()=>assert.equal(R.ATTACKS.find(x=>x.id==='truncate').invariant,false));
test('parse all 4416 evaluation records',()=>assert.equal(R.parse(data).length,4416));
test('complete manifest audit',()=>assert.equal(R.audit(data).ok,true));
test('no missing evaluation conditions',()=>assert.equal(R.audit(data).incomplete.length,0));
test('reject invalid model id',()=>assert.throws(()=>R.parse([{...data[0],method:'bogus'}])));
test('reject duplicate record keys',()=>assert.throws(()=>R.parse([data[0],data[0]])));
test('reject incorrect probability vector length',()=>assert.throws(()=>R.parse([{...data[0],scores:[.1,.2]}])));
test('unknown perturbation rejected',()=>assert.throws(()=>R.perturb('text','alien')));
test('format transforms text',()=>assert.notEqual(R.perturb('A statement.','format'),'A statement.'));
test('format preserves clean label on all examples for hybrid challenge',()=>close(R.summarize(data,'hybrid','challenge','format').flipRate,0));
test('typo deterministic',()=>assert.equal(R.perturb('Research findings demonstrate incremental change.','typo'),R.perturb('Research findings demonstrate incremental change.','typo')));
test('typo modifies eligible string',()=>assert.notEqual(R.perturb('Research findings demonstrate incremental change.','typo'),'Research findings demonstrate incremental change.'));
test('short token typo leaves unchanged',()=>assert.equal(R.perturb('a an is','typo'),'a an is'));
test('Spanish synonym substitution',()=>assert.notEqual(R.perturb('Este estudio es importante.','synonym','es'),'Este estudio es importante.'));
test('English synonym substitution',()=>assert.notEqual(R.perturb('Our research evaluates the results.','synonym','en'),'Our research evaluates the results.'));
test('Portuguese synonym substitution',()=>assert.notEqual(R.perturb('Este estudo apresenta resultados.','synonym','pt'),'Este estudo apresenta resultados.'));
test('unmapped dictionary leaves text unchanged',()=>assert.equal(R.perturb('Blue moon river','synonym','en'),'Blue moon river'));
test('context appended',()=>assert.ok(R.perturb('Hello.','context','en').includes('larger research discussion')));
test('truncation shortens long text',()=>assert.ok(R.perturb('one two three four five six seven eight nine ten','truncate').length< 'one two three four five six seven eight nine ten'.length));
test('full unchanged accuracy match (hybrid)',()=>close(R.summarize(data,'hybrid','challenge','original').accuracy,42/54));
test('multinomial NB challenge correct 40 of 54',()=>close(R.summarize(data,'nb','challenge','original').accuracy,40/54));
test('matched baseline subset differs from entire original in sparse synonym attack',()=>{let a=R.summarize(data,'nb','challenge','synonym');assert.equal(a.changed,9);assert.ok(a.baselineMatchedAccuracy!==R.summarize(data,'nb','challenge','original').accuracy);});
test('synonym flip denominator includes full split',()=>close(R.summarize(data,'hybrid','challenge','synonym').flipRate,1/54));
test('typo counts modified rows',()=>assert.equal(R.summarize(data,'hybrid','challenge','typo').changed,54));
test('destructive truncation never exposes inherited accuracy',()=>assert.equal(R.summarize(data,'hybrid','challenge','truncate').accuracy,null));
test('destructive truncation never exposes delta',()=>assert.equal(R.summarize(data,'hybrid','challenge','truncate').accuracyDelta,null));
test('risk curve ends at true overall error',()=>{const r=data.filter(x=>x.method==='nb'&&x.split==='challenge'&&x.attack==='original'),c=R.riskCoverage(r);close(c.at(-1).risk,1-R.accuracy(r));close(c.at(-1).coverage,1);});
test('risk curve has as many observations as predictions',()=>assert.equal(R.riskCoverage(data.filter(x=>x.method==='nb'&&x.split==='challenge'&&x.attack==='original')).length,54));
test('risk curve coverage is increasing',()=>{const c=R.riskCoverage(data.filter(x=>x.method==='nb'&&x.split==='challenge'&&x.attack==='original'));for(let i=1;i<c.length;i++)assert.ok(c[i].coverage>c[i-1].coverage);});
test('risk curve full coverage checkpoint',()=>{const r=data.filter(x=>x.method==='hybrid'&&x.split==='challenge'&&x.attack==='original');assert.equal(R.coverageAt(r,1).n,54)});
test('coverage input validation',()=>assert.throws(()=>R.coverageAt(data,.0)));
test('AURC bounded',()=>{const v=R.aurc(data.filter(x=>x.method==='hybrid'&&x.split==='challenge'&&x.attack==='original'));assert.ok(v>=0&&v<=1);});
test('Naive Bayes posterior probabilities sum to 1',()=>{const rows=data.filter(x=>x.method==='nb'&&x.split==='challenge'&&x.attack==='original');for(const r of rows)close(r.scores.reduce((a,b)=>a+b,0),1,1e-6);});
test('ECE bounded in 0..1',()=>{const x=R.ece(data.filter(x=>x.method==='nb'&&x.split==='challenge'&&x.attack==='original'));assert.ok(x>=0&&x<=1);});
test('reliability bins preserve support',()=>{const x=data.filter(x=>x.method==='nb'&&x.split==='challenge'&&x.attack==='original');assert.equal(R.reliability(x).reduce((a,b)=>a+b.n,0),x.length);});
test('reliability bins reject bad bin count',()=>assert.throws(()=>R.reliability(data,1)));
test('slices by language add to 54',()=>assert.equal(R.slices(data.filter(x=>x.method==='nb'&&x.split==='challenge'&&x.attack==='original'),'language').reduce((a,b)=>a+b.n,0),54));
test('unsupported slice key rejected',()=>assert.throws(()=>R.slices(data,'personal_secret')));
test('bootstrap repeatable by seed',()=>assert.deepEqual(R.pairedBootstrap(data,'hybrid','challenge','context',75,55),R.pairedBootstrap(data,'hybrid','challenge','context',75,55)));
test('bootstrap interval around observed point',()=>{const a=R.pairedBootstrap(data,'hybrid','challenge','context',300,23);assert.ok(a.low<=a.point&&a.point<=a.high);});
test('bootstrap refuses destructive score',()=>assert.throws(()=>R.pairedBootstrap(data,'nb','challenge','truncate')));
test('reject too few bootstrap replicates',()=>assert.throws(()=>R.pairedBootstrap(data,'nb','iid','typo',2)));
test('seeded random deterministic',()=>{const a=R.seeded(91),b=R.seeded(91);for(let i=0;i<15;i++)close(a(),b());});
test('default release gate applies explicit checks',()=>{const g=R.releaseGate(data,'hybrid','challenge');assert.equal(g.checks.length,9);assert.equal(typeof g.pass,'boolean')});
test('tightened gate cannot falsely pass',()=>{const a=R.releaseGate(data,'nb','challenge',{maxFlipRate:0,maxAccuracyDrop:0});assert.equal(a.pass,false);});
test('relaxed gate supports changed evaluation without claiming safety',()=>{const a=R.releaseGate(data,'hybrid','challenge',{minN:1,maxFlipRate:1,maxAccuracyDrop:1});assert.equal(a.pass,true);assert.ok(a.note.includes('not production'))});
test('safe CSV quotes commas and double-quotes',()=>assert.ok(R.csv([{v:'alpha,"beta"'}],['v']).includes('"alpha,""beta"""')));
test('safe CSV neutralizes formula injection',()=>assert.ok(R.csv([{v:'=SUM(1,2)'}],['v']).includes("'=SUM")));
test('original source model reproduces 77.8 percent hybrid challenge',()=>{const bundle=M.train(train);const p=M.evaluate(bundle,challenge,'hybrid');close(M.evaluation(p).accuracy,42/54);});
test('snapshot seed data preserve identity',()=>{const keys=new Set(challenge.map(x=>x.id));assert.ok(data.filter(x=>x.split==='challenge').every(x=>keys.has(x.id)));});
console.log('TOTAL ENGINE TESTS',pass,process.exitCode?'FAIL':'PASS');