Spaces:
Running
Running
| /* RobustnessLab AI — reproducible evaluation and stress-test utilities. MIT. */ | |
| (function(root,factory){const api=factory();if(typeof module==='object'&&module.exports)module.exports=api;root.RobustnessEngine=api;})(typeof globalThis!=='undefined'?globalThis:this,function(){'use strict'; | |
| const ATTACKS=[ | |
| {id:'original',name:'Unchanged original',level:'reference',invariant:true,meaning:'Original evaluation statement'}, | |
| {id:'format',name:'Case and punctuation',level:'low',invariant:true,meaning:'Surface-level formatting, presumed label stable'}, | |
| {id:'typo',name:'Deterministic typo',level:'medium',invariant:true,meaning:'A single character transposition; presumption requires review'}, | |
| {id:'synonym',name:'Dictionary substitution',level:'medium',invariant:true,meaning:'Heuristic word replacement; meaning may shift and requires review'}, | |
| {id:'context',name:'Neutral context sentence',level:'medium',invariant:true,meaning:'Additional context; no true semantic invariance guarantee'}, | |
| {id:'truncate',name:'Information removal',level:'destructive',invariant:false,meaning:'Deletion may change the gold label; no accuracy claim'} | |
| ]; | |
| const METHOD_NAMES={nb:'Multinomial Naive Bayes',word:'Word TF-IDF',char:'Character TF-IDF',hybrid:'Hybrid TF-IDF'}; | |
| const METHODS=Object.keys(METHOD_NAMES); | |
| const LABELS=['background','objective','method','result','limitation','implication']; | |
| function stableHash(s){let h=2166136261;for(let i=0;i<String(s).length;i++){h^=String(s).charCodeAt(i);h=Math.imul(h,16777619);}return h>>>0;} | |
| function pickIndex(text,words){const indices=[];for(let i=0;i<words.length;i++)if(/^[\p{L}]{5,}$/u.test(words[i]))indices.push(i);if(!indices.length)return -1;return indices[stableHash(text)%indices.length];} | |
| function typo(text){const chunks=String(text).split(/(\s+)/);const wordIndices=chunks.map((s,i)=>(i%2===0&&/^[\p{L}]{5,}[.,;:!?]?$/u.test(s))?i:-1).filter(x=>x>=0);if(!wordIndices.length)return text;const idx=wordIndices[stableHash(text)%wordIndices.length],w=chunks[idx];const letters=Array.from(w);const positions=[];for(let j=1;j<letters.length-2;j++)if(/\p{L}/u.test(letters[j])&&/\p{L}/u.test(letters[j+1])&&letters[j]!==letters[j+1])positions.push(j);if(!positions.length)return text;const k=positions[stableHash(text+'edit')%positions.length];[letters[k],letters[k+1]]=[letters[k+1],letters[k]];chunks[idx]=letters.join('');return chunks.join('');} | |
| const REPLACEMENTS={ | |
| en:[['study','investigation'],['research','analysis'],['results','findings'],['analysis','examination'],['evidence','indications'],['data','information'],['evaluate','assess'],['propose','suggest'],['changes','shifts'],['important','significant'],['shows','indicates'],['however','nevertheless'],['method','approach'],['effect','impact'],['using','employing'],['impact','influence'],['question','issue'],['limits','constraints'],['objective','aim']], | |
| es:[['estudio','investigación'],['investigación','análisis'],['resultados','hallazgos'],['análisis','examen'],['datos','información'],['evaluar','examinar'],['proponer','plantear'],['cambios','variaciones'],['importante','relevante'],['muestra','indica'],['sin embargo','no obstante'],['método','enfoque'],['efecto','impacto'],['utilizando','empleando'],['objetivo','propósito'],['limitaciones','restricciones'],['evidencia','indicios']], | |
| pt:[['estudo','pesquisa'],['pesquisa','análise'],['resultados','achados'],['análise','exame'],['dados','informações'],['avaliar','examinar'],['propor','sugerir'],['mudanças','alterações'],['importante','relevante'],['mostra','indica'],['entretanto','contudo'],['método','abordagem'],['efeito','impacto'],['utilizando','empregando'],['objetivo','propósito'],['limitações','restrições'],['evidência','indícios']] | |
| }; | |
| function synonym(text,lang){const list=REPLACEMENTS[lang]||REPLACEMENTS.en;const possibles=[];for(const [from,to] of list){const re=new RegExp('(?<![\\p{L}])'+from.replace(/[.*+?^${}()|[\]\\]/g,'\\$&')+'(?![\\p{L}])','iu');if(re.test(text))possibles.push({from,to,re});}if(!possibles.length)return text;const p=possibles[stableHash(text)%possibles.length];return String(text).replace(p.re,m=>m[0]===m[0].toUpperCase()?p.to[0].toUpperCase()+p.to.slice(1):p.to);} | |
| function context(text,lang){const s={en:'This statement is part of a larger research discussion.',es:'Esta afirmación forma parte de una discusión de investigación más amplia.',pt:'Esta afirmação faz parte de um debate de pesquisa mais amplo.'}[lang]||'This text is part of a larger discussion.';return String(text).trim()+' '+s;} | |
| function trunc(text){const words=String(text).trim().split(/\s+/);if(words.length<=3)return text;return words.slice(0,Math.max(2,Math.floor(words.length*.65))).join(' ').replace(/[,;:]$/,'')+'.';} | |
| function perturb(text,attack,lang='en'){const s=String(text??'');switch(attack){case 'original':return s;case 'format':return '“'+s.toUpperCase().replace(/[.!?]+$/,'')+'!”';case 'typo':return typo(s);case 'synonym':return synonym(s,lang);case 'context':return context(s,lang);case 'truncate':return trunc(s);default:throw Error('Unknown perturbation '+attack);}} | |
| function parse(rows){if(!Array.isArray(rows)||!rows.length||rows.length>20000)throw Error('Expected 1–20,000 robustness records');const seen=new Set();return rows.map((r,i)=>{if(!r||!METHODS.includes(r.method)||!['iid','challenge'].includes(r.split)||!ATTACKS.some(a=>a.id===r.attack)||!LABELS.includes(r.truth)||!LABELS.includes(r.pred)||!LABELS.includes(r.originalPred)||!['en','es','pt'].includes(r.language)||!r.id)throw Error('Invalid record '+i);const id=[r.method,r.split,r.attack,r.id].join('|');if(seen.has(id))throw Error('Duplicate '+id);seen.add(id);const scores=Array.isArray(r.scores)?r.scores.map(Number):[];if(scores.length!==LABELS.length||scores.some(v=>!Number.isFinite(v)))throw Error('Invalid prediction scores '+id);const margin=Number(r.margin);if(!Number.isFinite(margin)||margin<0)throw Error('Invalid margin '+id);return {...r,scores,margin,changed:Boolean(r.changed)};});} | |
| const division=(a,b)=>b?a/b:null; | |
| function accuracy(rows){return division(rows.filter(r=>r.pred===r.truth).length,rows.length);} | |
| function summarize(rows,model,split,attack){const selected=rows.filter(r=>r.method===model&&r.split===split&&r.attack===attack);const changed=selected.filter(r=>r.changed||attack==='original');const flips=selected.filter(r=>r.pred!==r.originalPred).length;const base=changed.filter(r=>r.originalPred===r.truth).length;const scored=ATTACKS.find(a=>a.id===attack).invariant;const acc=scored?accuracy(changed):null;return {method:model,split,attack,n:selected.length,changed:changed.length,unchanged:selected.length-changed.length,changedShare:division(changed.length,selected.length),flips,flipRate:division(flips,selected.length),changedFlipRate:division(changed.filter(r=>r.pred!==r.originalPred).length,changed.length),accuracy:acc,baselineMatchedAccuracy:scored?division(base,changed.length):null,accuracyDelta:scored&&acc!=null?acc-division(base,changed.length):null,scored};} | |
| function quantile(sorted,q){return sorted[Math.max(0,Math.min(sorted.length-1,Math.floor(q*(sorted.length-1))))];} | |
| function riskCoverage(rows){if(!rows.length)return [];const sorted=[...rows].sort((a,b)=>b.margin-a.margin||a.id.localeCompare(b.id));let errors=0;return sorted.map((r,i)=>{errors+=r.pred!==r.truth?1:0;return {coverage:(i+1)/sorted.length,risk:errors/(i+1),n:i+1,threshold:r.margin};});} | |
| function aurc(rows){const curve=riskCoverage(rows);return curve.length?curve.reduce((s,p)=>s+p.risk,0)/curve.length:null;} | |
| function coverageAt(rows,minCoverage){if(!Number.isFinite(minCoverage)||minCoverage<=0||minCoverage>1)throw Error('Coverage must be in (0,1]');const curve=riskCoverage(rows);return curve[Math.max(0,Math.ceil(curve.length*minCoverage)-1)]||null;} | |
| function reliability(rows,bins=8){if(!rows.length)return [];if(!Number.isInteger(bins)||bins<2||bins>20)throw Error('Invalid bin count');const out=Array.from({length:bins},(_,i)=>({low:i/bins,high:(i+1)/bins,n:0,correct:0,confidenceSum:0}));for(const r of rows){const confidence=Math.max(...r.scores);const idx=Math.min(bins-1,Math.floor(confidence*bins));out[idx].n++;out[idx].correct+=r.pred===r.truth?1:0;out[idx].confidenceSum+=confidence;}return out.map(x=>({...x,empiricalAccuracy:division(x.correct,x.n),meanConfidence:division(x.confidenceSum,x.n)}));} | |
| function ece(rows,bins=8){if(!rows.length)return null;return reliability(rows,bins).reduce((sum,b)=>sum+(b.n/rows.length)*Math.abs((b.empiricalAccuracy??0)-(b.meanConfidence??0)),0);} | |
| function slices(rows,key){if(!['language','domain','truth'].includes(key))throw Error('Unsupported grouping');const map=new Map();for(const r of rows){const v=String(r[key]||'other');if(!map.has(v))map.set(v,[]);map.get(v).push(r);}return Array.from(map,([name,x])=>({name,n:x.length,accuracy:accuracy(x),flips:x.filter(r=>r.pred!==r.originalPred).length,flipRate:division(x.filter(r=>r.pred!==r.originalPred).length,x.length)})).sort((a,b)=>a.name.localeCompare(b.name));} | |
| function seeded(seed){let t=(Number(seed)||1)>>>0;return()=>{t+=0x6D2B79F5;let x=t;x=Math.imul(x^(x>>>15),x|1);x^=x+Math.imul(x^(x>>>7),x|61);return((x^(x>>>14))>>>0)/4294967296;};} | |
| function pairedBootstrap(rows,model,split,attack,iterations=400,seed=124){const matched=rows.filter(r=>r.method===model&&r.split===split&&r.attack===attack&&r.changed);if(!ATTACKS.find(a=>a.id===attack).invariant)throw Error('Do not score destructive perturbations');if(matched.length<2)return null;if(iterations<50||iterations>3000)throw Error('Invalid iterations');const deltas=matched.map(r=>Number(r.pred===r.truth)-Number(r.originalPred===r.truth));const rng=seeded(seed),arr=[];for(let i=0;i<iterations;i++){let s=0;for(let j=0;j<deltas.length;j++)s+=deltas[Math.floor(rng()*deltas.length)];arr.push(s/deltas.length);}arr.sort((a,b)=>a-b);return {n:deltas.length,point:deltas.reduce((a,b)=>a+b,0)/deltas.length,low:quantile(arr,.025),high:quantile(arr,.975),iterations,seed};} | |
| function audit(rows){const parsed=parse(rows);const methods=new Set(parsed.map(r=>r.method)),sets=new Set(parsed.map(r=>r.split)),attacks=new Set(parsed.map(r=>r.attack));const incomplete=[];for(const method of methods)for(const split of sets){const baseline=parsed.filter(r=>r.method===method&&r.split===split&&r.attack==='original');const ids=new Set(baseline.map(r=>r.id));for(const att of attacks){const group=parsed.filter(r=>r.method===method&&r.split===split&&r.attack===att);if(group.length!==baseline.length||group.some(r=>!ids.has(r.id)))incomplete.push(method+' / '+split+' / '+att);}}return {n:parsed.length,methods:[...methods].sort(),splits:[...sets].sort(),attacks:[...attacks].sort(),incomplete,ok:incomplete.length===0};} | |
| function escapeCsv(value){let s=String(value??'');if(/^[\s]*[=+@\-\t\r]/.test(s))s="'"+s;return /[",\r\n]/.test(s)?'"'+s.replace(/"/g,'""')+'"':s;} | |
| function csv(rows,keys){return [keys.join(','),...rows.map(r=>keys.map(k=>escapeCsv(r[k])).join(','))].join('\r\n');} | |
| function releaseGate(rows,model='hybrid',split='challenge',opts={}){const rules={minN:40,maxFlipRate:.15,maxAccuracyDrop:.10,minAURCRatio:0, ...opts};const original=rows.filter(r=>r.method===model&&r.split===split&&r.attack==='original');const all=ATTACKS.filter(a=>a.invariant&&a.id!=='original').map(a=>summarize(rows,model,split,a.id));const checks=[{rule:'Minimum original N',pass:original.length>=rules.minN,actual:original.length,threshold:rules.minN}];for(const result of all){checks.push({rule:result.attack+' flip rate',pass:result.flipRate!=null&&result.flipRate<=rules.maxFlipRate,actual:result.flipRate,threshold:rules.maxFlipRate});checks.push({rule:result.attack+' matched accuracy drop',pass:result.accuracyDelta!=null&&result.accuracyDelta>=-rules.maxAccuracyDrop,actual:result.accuracyDelta,threshold:-rules.maxAccuracyDrop});}return {model,split,pass:checks.every(r=>r.pass),checks,note:'Exploratory research gates, not production approval or safety certification.'};} | |
| return {ATTACKS,METHODS,METHOD_NAMES,LABELS,perturb,parse,summarize,accuracy,riskCoverage,aurc,coverageAt,reliability,ece,slices,seeded,pairedBootstrap,audit,csv,releaseGate}; | |
| }); | |