Spaces:
Running
Running
| ; | |
| const assert=require('node:assert/strict'),fs=require('node:fs'),path=require('node:path'),E=require('./evalflow_engine.js'); | |
| const pathRoot=path.resolve(__dirname);const source=JSON.parse(fs.readFileSync(path.join(pathRoot,'robustness_snapshots.json'),'utf8')); | |
| const rows=source.rows;let checks=0; | |
| function test(name,fn){fn();checks++;console.log('PASS',name)} | |
| function close(a,b,eps=1e-10){assert(Math.abs(a-b)<eps,`${a} != ${b}`)} | |
| function cloned(){return rows.map(x=>({...x}));} | |
| test('complete original attack sets',()=>assert(E.completeEvidence(rows,'hybrid','challenge').complete)); | |
| test('duplicate-count swapped ID blocks integrity',()=>{const a=cloned(),r=a.find(x=>x.method==='hybrid'&&x.split==='challenge'&&x.attack==='typo');r.id='invented-001';assert(!E.completeEvidence(a,'hybrid','challenge').complete)}); | |
| test('inconsistent originalPred blocks integrity',()=>{const a=cloned(),r=a.find(x=>x.method==='hybrid'&&x.split==='challenge'&&x.attack==='context');r.originalPred='background';assert(!E.completeEvidence(a,'hybrid','challenge').complete)}); | |
| test('source has 4416 prediction records',()=>assert.equal(rows.length,4416)); | |
| test('source metadata is synthetic-curated',()=>assert.equal(source.metadata.sourceType,'synthetic-curated')); | |
| test('schema passes on original source',()=>E.verify(rows)); | |
| test('duplicate record rejected',()=>assert.throws(()=>E.verify([rows[0],rows[0]]),/Duplicate/)); | |
| test('bad method rejected',()=>assert.throws(()=>E.verify([{...rows[0],method:'llm'}]),/Unknown classifier/)); | |
| test('bad label rejected',()=>assert.throws(()=>E.verify([{...rows[0],truth:'unknown'}]),/Unknown rhetorical/)); | |
| test('bad language rejected',()=>assert.throws(()=>E.verify([{...rows[0],language:'xx'}]),/Unknown language/)); | |
| test('missing ID rejected',()=>assert.throws(()=>E.verify([{...rows[0],id:''}]),/Invalid example ID/)); | |
| test('empty records rejected',()=>assert.throws(()=>E.verify([]),/Expected/)); | |
| test('reject malformed selection',()=>assert.throws(()=>E.get(rows,'garbage','iid'),/Invalid selection/)); | |
| test('labels are six rhetorical classes',()=>assert.equal(E.LABELS.length,6)); | |
| test('baseline original IID count 130',()=>assert.equal(E.get(rows,'nb','iid').length,130)); | |
| test('challenge original count 54',()=>assert.equal(E.get(rows,'nb','challenge').length,54)); | |
| test('original challenge NB accuracy 74.1 percent',()=>close(E.dataset(rows,'nb','challenge').accuracy,40/54)); | |
| test('hybrid challenge accuracy 77.8 percent',()=>close(E.dataset(rows,'hybrid','challenge').accuracy,42/54)); | |
| test('hybrid IID accuracy 97.7 percent',()=>close(E.dataset(rows,'hybrid','iid').accuracy,127/130)); | |
| test('macro F1 is between 0 and 1',()=>{const f=E.dataset(rows,'hybrid','challenge').macroF1;assert(f>0&&f<1)}); | |
| test('per-class F1 has 6 valid counts',()=>{const x=E.dataset(rows,'hybrid','challenge').classes;assert.equal(x.length,6);assert.equal(x.reduce((s,a)=>s+a.n,0),54)}); | |
| test('all three challenge languages have 18 records',()=>{const x=E.dataset(rows,'hybrid','challenge').groups.language;assert.equal(x.length,3);assert(x.every(z=>z.n===18));}); | |
| test('slice group supported domain',()=>assert(E.byGroup(E.get(rows,'nb','iid'),'domain').length>1)); | |
| test('unsupported slice rejected',()=>assert.throws(()=>E.byGroup(rows,'userEmail'),/Unsupported/)); | |
| test('six attacks each contain 54 examples',()=>assert(E.dataset(rows,'nb','challenge').attacks.every(a=>a.n===54))); | |
| test('truncation accuracy not scored',()=>assert.equal(E.dataset(rows,'hybrid','challenge').attacks.find(x=>x.id==='truncate').matchedAccuracy,null)); | |
| test('hybrid typo changes few original predictions',()=>close(E.dataset(rows,'hybrid','challenge').attacks.find(x=>x.id==='typo').flipRate,1/54)); | |
| test('hybrid context flips five of 54',()=>close(E.dataset(rows,'hybrid','challenge').attacks.find(x=>x.id==='context').flipRate,5/54)); | |
| test('paired comparison uses same 54 example IDs',()=>{const p=E.paired(rows,'nb','hybrid','challenge');assert.equal(p.n,54);assert(p.complete)}); | |
| test('paired improved vs regressed counts consistent',()=>{const p=E.paired(rows,'nb','hybrid','challenge');close(p.delta,(p.improved-p.regressed)/p.n)}); | |
| test('paired original is immutable',()=>{const p=E.paired(rows,'nb','hybrid','iid');assert.equal(p.n,130)}); | |
| test('seeded bootstrap reproducible',()=>{const p=E.paired(rows,'nb','hybrid','challenge');assert.deepEqual(E.bootstrap(p),E.bootstrap(p))}); | |
| test('different bootstrap seed allowed',()=>{const p=E.paired(rows,'nb','hybrid','challenge');assert.equal(E.bootstrap(p,{iterations:150,seed:1}).iterations,150)}); | |
| test('too few bootstrap repetitions rejected',()=>assert.throws(()=>E.bootstrap(E.paired(rows,'nb','hybrid','challenge'),{iterations:50}),/Bad iteration/)); | |
| test('balanced release gate PASS',()=>assert.equal(E.evaluate(rows,'nb','hybrid','challenge').status,'PASS')); | |
| test('word -> hybrid on challenge PASS',()=>assert.equal(E.evaluate(rows,'word','hybrid','challenge').status,'PASS')); | |
| test('regression candidate nb from hybrid BLOCK',()=>assert.equal(E.evaluate(rows,'hybrid','nb','challenge').status,'BLOCK')); | |
| test('tight threshold blocks',()=>assert.equal(E.evaluate(rows,'nb','hybrid','challenge',{minAccuracy:.85}).status,'BLOCK')); | |
| test('zero tolerance context flip blocks',()=>assert.equal(E.evaluate(rows,'nb','hybrid','challenge',{maxContextFlip:0}).status,'BLOCK')); | |
| test('language N threshold 19 blocks challenge',()=>assert.equal(E.evaluate(rows,'nb','hybrid','challenge',{minLanguageN:19}).status,'BLOCK')); | |
| test('strict minimum subgroup Accuracy blocks',()=>assert.equal(E.evaluate(rows,'nb','hybrid','challenge',{minLanguageAccuracy:.85}).status,'BLOCK')); | |
| test('controlled split selected',()=>assert.equal(E.evaluate(rows,'nb','hybrid','iid').candidateMetrics.n,130)); | |
| test('same method comparison rejected',()=>assert.throws(()=>E.evaluate(rows,'hybrid','hybrid'),/different models/)); | |
| test('bad regression tolerance rejected',()=>assert.throws(()=>E.policyCheck({maxRegression:-1}),/Invalid policy/)); | |
| test('max accuracy > 1 rejected',()=>assert.throws(()=>E.policyCheck({minAccuracy:2}),/<= 1/)); | |
| test('fractional minimum sample rejected',()=>assert.throws(()=>E.policyCheck({minN:1.5}),/whole numbers/)); | |
| test('bad bool policy rejected',()=>assert.throws(()=>E.policyCheck({requireComplete:'yes'}),/boolean/)); | |
| test('non object policy rejected',()=>assert.throws(()=>E.policyCheck([]),/object/)); | |
| test('paired label mismatch blocked',()=>{const changed=cloned();const t=changed.find(x=>x.method==='hybrid'&&x.split==='challenge'&&x.attack==='original');t.truth=t.truth==='background'?'objective':'background';assert.throws(()=>E.paired(changed,'nb','hybrid','challenge'),/Mismatched/)}); | |
| test('missing original candidate sample blocks',()=>{const changed=rows.filter(x=>!(x.method==='hybrid'&&x.split==='challenge'&&x.attack==='original'&&x.id===E.get(rows,'hybrid','challenge')[0].id));const r=E.evaluate(changed,'nb','hybrid','challenge');assert.equal(r.status,'BLOCK');assert.equal(r.pair.onlyBase.length,1);}); | |
| test('missing stress condition blocks',()=>{const changed=rows.filter(x=>!(x.method==='hybrid'&&x.split==='challenge'&&x.attack==='typo'&&x.id===E.get(rows,'hybrid','challenge','typo')[0].id));assert.equal(E.evaluate(changed,'nb','hybrid','challenge').status,'BLOCK');}); | |
| test('manual CSV strings protected against Excel formula',()=>assert(E.escapeCsv('=HYPERLINK("bad")').includes("'="))); | |
| test('CSV quotes escaped',()=>assert(E.csv([{a:'a,"b"'}],['a']).includes('"a,""b"""'))); | |
| test('report includes transparent limitations',()=>assert(E.report(E.evaluate(rows,'nb','hybrid')).toLowerCase().includes('synthetic'))); | |
| test('report gates show actual status',()=>assert(E.report(E.evaluate(rows,'hybrid','nb')).includes('**BLOCK**'))); | |
| test('risk message does not claim production certification',()=>assert(E.evaluate(rows,'nb','hybrid').note.includes('not production'))); | |
| const cli=require('./ci_evalflow.js'); | |
| test('CLI args parser accepts known flags',()=>assert.equal(cli.argsFrom(['--baseline=nb']).baseline,'nb')); | |
| test('CLI rejects unknown flag',()=>assert.throws(()=>cli.argsFrom(['--privateToken=abc']),/Unknown flag/)); | |
| test('no raw secrets required',()=>assert(!JSON.stringify(E.DEFAULT_POLICY).includes('token'))); | |
| console.log('TOTAL ANALYTICAL TESTS',checks,'PASS'); | |