Spaces:
Running
Running
| const assert=require('node:assert/strict'),fs=require('fs'); | |
| const R=require('./robustness_engine'),M=require('./source_modelops_engine'); | |
| const data=JSON.parse(fs.readFileSync('robustness_snapshots.json','utf8')).rows; | |
| const train=JSON.parse(fs.readFileSync('train.json','utf8')); | |
| const iid=JSON.parse(fs.readFileSync('iid_test.json','utf8')); | |
| const challenge=JSON.parse(fs.readFileSync('challenge.json','utf8')); | |
| let pass=0; | |
| const test=(title,fn)=>{try{fn();pass++;process.stdout.write('PASS '+title+'\n');}catch(err){console.error('FAIL '+title,err);process.exitCode=1;}}; | |
| function close(a,b,eps=1e-9){assert.ok(Math.abs(a-b)<eps,`${a} != ${b}`)} | |
| test('original 842 training examples preserved',()=>assert.equal(train.length,842)); | |
| test('original 130 controlled examples preserved',()=>assert.equal(iid.length,130)); | |
| test('original 54 challenge examples preserved',()=>assert.equal(challenge.length,54)); | |
| test('four methods registered',()=>assert.deepEqual(R.METHODS,['nb','word','char','hybrid'])); | |
| test('six test conditions registered',()=>assert.equal(R.ATTACKS.length,6)); | |
| test('truncate explicitly noninvariant',()=>assert.equal(R.ATTACKS.find(x=>x.id==='truncate').invariant,false)); | |
| test('parse all 4416 evaluation records',()=>assert.equal(R.parse(data).length,4416)); | |
| test('complete manifest audit',()=>assert.equal(R.audit(data).ok,true)); | |
| test('no missing evaluation conditions',()=>assert.equal(R.audit(data).incomplete.length,0)); | |
| test('reject invalid model id',()=>assert.throws(()=>R.parse([{...data[0],method:'bogus'}]))); | |
| test('reject duplicate record keys',()=>assert.throws(()=>R.parse([data[0],data[0]]))); | |
| test('reject incorrect probability vector length',()=>assert.throws(()=>R.parse([{...data[0],scores:[.1,.2]}]))); | |
| test('unknown perturbation rejected',()=>assert.throws(()=>R.perturb('text','alien'))); | |
| test('format transforms text',()=>assert.notEqual(R.perturb('A statement.','format'),'A statement.')); | |
| test('format preserves clean label on all examples for hybrid challenge',()=>close(R.summarize(data,'hybrid','challenge','format').flipRate,0)); | |
| test('typo deterministic',()=>assert.equal(R.perturb('Research findings demonstrate incremental change.','typo'),R.perturb('Research findings demonstrate incremental change.','typo'))); | |
| test('typo modifies eligible string',()=>assert.notEqual(R.perturb('Research findings demonstrate incremental change.','typo'),'Research findings demonstrate incremental change.')); | |
| test('short token typo leaves unchanged',()=>assert.equal(R.perturb('a an is','typo'),'a an is')); | |
| test('Spanish synonym substitution',()=>assert.notEqual(R.perturb('Este estudio es importante.','synonym','es'),'Este estudio es importante.')); | |
| test('English synonym substitution',()=>assert.notEqual(R.perturb('Our research evaluates the results.','synonym','en'),'Our research evaluates the results.')); | |
| test('Portuguese synonym substitution',()=>assert.notEqual(R.perturb('Este estudo apresenta resultados.','synonym','pt'),'Este estudo apresenta resultados.')); | |
| test('unmapped dictionary leaves text unchanged',()=>assert.equal(R.perturb('Blue moon river','synonym','en'),'Blue moon river')); | |
| test('context appended',()=>assert.ok(R.perturb('Hello.','context','en').includes('larger research discussion'))); | |
| test('truncation shortens long text',()=>assert.ok(R.perturb('one two three four five six seven eight nine ten','truncate').length< 'one two three four five six seven eight nine ten'.length)); | |
| test('full unchanged accuracy match (hybrid)',()=>close(R.summarize(data,'hybrid','challenge','original').accuracy,42/54)); | |
| test('multinomial NB challenge correct 40 of 54',()=>close(R.summarize(data,'nb','challenge','original').accuracy,40/54)); | |
| test('matched baseline subset differs from entire original in sparse synonym attack',()=>{let a=R.summarize(data,'nb','challenge','synonym');assert.equal(a.changed,9);assert.ok(a.baselineMatchedAccuracy!==R.summarize(data,'nb','challenge','original').accuracy);}); | |
| test('synonym flip denominator includes full split',()=>close(R.summarize(data,'hybrid','challenge','synonym').flipRate,1/54)); | |
| test('typo counts modified rows',()=>assert.equal(R.summarize(data,'hybrid','challenge','typo').changed,54)); | |
| test('destructive truncation never exposes inherited accuracy',()=>assert.equal(R.summarize(data,'hybrid','challenge','truncate').accuracy,null)); | |
| test('destructive truncation never exposes delta',()=>assert.equal(R.summarize(data,'hybrid','challenge','truncate').accuracyDelta,null)); | |
| test('risk curve ends at true overall error',()=>{const r=data.filter(x=>x.method==='nb'&&x.split==='challenge'&&x.attack==='original'),c=R.riskCoverage(r);close(c.at(-1).risk,1-R.accuracy(r));close(c.at(-1).coverage,1);}); | |
| test('risk curve has as many observations as predictions',()=>assert.equal(R.riskCoverage(data.filter(x=>x.method==='nb'&&x.split==='challenge'&&x.attack==='original')).length,54)); | |
| test('risk curve coverage is increasing',()=>{const c=R.riskCoverage(data.filter(x=>x.method==='nb'&&x.split==='challenge'&&x.attack==='original'));for(let i=1;i<c.length;i++)assert.ok(c[i].coverage>c[i-1].coverage);}); | |
| test('risk curve full coverage checkpoint',()=>{const r=data.filter(x=>x.method==='hybrid'&&x.split==='challenge'&&x.attack==='original');assert.equal(R.coverageAt(r,1).n,54)}); | |
| test('coverage input validation',()=>assert.throws(()=>R.coverageAt(data,.0))); | |
| test('AURC bounded',()=>{const v=R.aurc(data.filter(x=>x.method==='hybrid'&&x.split==='challenge'&&x.attack==='original'));assert.ok(v>=0&&v<=1);}); | |
| test('Naive Bayes posterior probabilities sum to 1',()=>{const rows=data.filter(x=>x.method==='nb'&&x.split==='challenge'&&x.attack==='original');for(const r of rows)close(r.scores.reduce((a,b)=>a+b,0),1,1e-6);}); | |
| test('ECE bounded in 0..1',()=>{const x=R.ece(data.filter(x=>x.method==='nb'&&x.split==='challenge'&&x.attack==='original'));assert.ok(x>=0&&x<=1);}); | |
| test('reliability bins preserve support',()=>{const x=data.filter(x=>x.method==='nb'&&x.split==='challenge'&&x.attack==='original');assert.equal(R.reliability(x).reduce((a,b)=>a+b.n,0),x.length);}); | |
| test('reliability bins reject bad bin count',()=>assert.throws(()=>R.reliability(data,1))); | |
| test('slices by language add to 54',()=>assert.equal(R.slices(data.filter(x=>x.method==='nb'&&x.split==='challenge'&&x.attack==='original'),'language').reduce((a,b)=>a+b.n,0),54)); | |
| test('unsupported slice key rejected',()=>assert.throws(()=>R.slices(data,'personal_secret'))); | |
| test('bootstrap repeatable by seed',()=>assert.deepEqual(R.pairedBootstrap(data,'hybrid','challenge','context',75,55),R.pairedBootstrap(data,'hybrid','challenge','context',75,55))); | |
| test('bootstrap interval around observed point',()=>{const a=R.pairedBootstrap(data,'hybrid','challenge','context',300,23);assert.ok(a.low<=a.point&&a.point<=a.high);}); | |
| test('bootstrap refuses destructive score',()=>assert.throws(()=>R.pairedBootstrap(data,'nb','challenge','truncate'))); | |
| test('reject too few bootstrap replicates',()=>assert.throws(()=>R.pairedBootstrap(data,'nb','iid','typo',2))); | |
| test('seeded random deterministic',()=>{const a=R.seeded(91),b=R.seeded(91);for(let i=0;i<15;i++)close(a(),b());}); | |
| test('default release gate applies explicit checks',()=>{const g=R.releaseGate(data,'hybrid','challenge');assert.equal(g.checks.length,9);assert.equal(typeof g.pass,'boolean')}); | |
| test('tightened gate cannot falsely pass',()=>{const a=R.releaseGate(data,'nb','challenge',{maxFlipRate:0,maxAccuracyDrop:0});assert.equal(a.pass,false);}); | |
| test('relaxed gate supports changed evaluation without claiming safety',()=>{const a=R.releaseGate(data,'hybrid','challenge',{minN:1,maxFlipRate:1,maxAccuracyDrop:1});assert.equal(a.pass,true);assert.ok(a.note.includes('not production'))}); | |
| test('safe CSV quotes commas and double-quotes',()=>assert.ok(R.csv([{v:'alpha,"beta"'}],['v']).includes('"alpha,""beta"""'))); | |
| test('safe CSV neutralizes formula injection',()=>assert.ok(R.csv([{v:'=SUM(1,2)'}],['v']).includes("'=SUM"))); | |
| test('original source model reproduces 77.8 percent hybrid challenge',()=>{const bundle=M.train(train);const p=M.evaluate(bundle,challenge,'hybrid');close(M.evaluation(p).accuracy,42/54);}); | |
| test('snapshot seed data preserve identity',()=>{const keys=new Set(challenge.map(x=>x.id));assert.ok(data.filter(x=>x.split==='challenge').every(x=>keys.has(x.id)));}); | |
| console.log('TOTAL ENGINE TESTS',pass,process.exitCode?'FAIL':'PASS'); | |