/* Relapse: everything below is drawn from data/results.json. No number or result sentence is written here. */ (function () { 'use strict'; const $ = (id) => document.getElementById(id); const el = (tag, cls, html) => { const e = document.createElement(tag); if (cls) e.className = cls; if (html != null) e.innerHTML = html; return e; }; const esc = (s) => String(s).replace(/[&<>"]/g, (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"' }[c])); const pct = (x) => Math.round(x * 100); const p1 = (x) => (x * 100).toFixed(1); const sgn = (x) => (x < 0 ? '−' : '+') + Math.abs(x * 100).toFixed(1); const clamp = (x, a, b) => Math.max(a, Math.min(b, x)); const reduced = () => window.matchMedia('(prefers-reduced-motion: reduce)').matches; const RISE = 0.05, GREW = 0.02, PAID = 0.03; // a relapse is +5 points over fp16; the page turns red when a file loses 3+ points of general knowledge let D, FP, CH, ORIG, NOTORIG, FMT = {}, MET = {}, LANG = {}, STATS = {}, DEF = {}; let st = { m: 'standard', f: 'NF4', lang: '' }; let lastNum = null, built = false, topCache = {}; fetch('data/results.json', { cache: 'no-cache' }).then((r) => r.json()).then(init).catch(() => { $('line').textContent = 'The recorded run could not be loaded. Reload the page to try again.'; }); const cell = (m, f) => (D.grid[m] && D.grid[m][f]) || null; const fmtLabel = (f) => (FMT[f] ? FMT[f].label : f); const methodLabel = (m) => (MET[m] ? MET[m].label : m); const methodShort = (m) => (MET[m] ? (MET[m].short || MET[m].label) : m); const forgetOf = (c, lang) => (!c ? null : lang ? (c.probes && c.probes[lang] != null ? c.probes[lang] : null) : c.forget); const langName = (l) => (LANG[l] || l); /* paired AuditKIT rows: forget_holds rows say method + from + to, price rows say method + prec */ function indexStats() { const near = (a, b) => a != null && b != null && Math.abs(a - b) < 0.004; (D.stats || []).forEach((s) => { if (s.kind === 'price') { if (s.method && s.prec) STATS['p|' + s.method + '|' + s.prec] = s; return; } if (s.method && s.to) { STATS['f|' + s.method + '|' + s.to] = s; return; } if (!s.a || !s.b) return; const fA = s.a.slice(s.a.lastIndexOf('-') + 1), fB = s.b.slice(s.b.lastIndexOf('-') + 1); Object.keys(D.grid).forEach((m) => { const ca = cell(m, fA), cb = cell(m, fB); if (ca && cb && near(ca.forget, s.acc_a) && near(cb.forget, s.acc_b)) STATS['f|' + m + '|' + fB] = s; }); }); } /* What the selection means, computed only from the grid. */ function analyse(m, f, lang) { const c = cell(m, f), a = { m, f, lang }; const v = forgetOf(c, lang); if (!c || v == null) { a.kind = 'missing'; a.field = 'known'; return a; } const o = (!lang && c.orig) || cell(ORIG, f), oV = forgetOf(o, lang); // the original scored in the same run, when the run had its own a.v = v; a.r = c.retain; a.o = oV; a.oR = o ? o.retain : null; a.pc = c.forget_pcorrect; const top = topOf(lang); a.rev = clamp((v - CH) / Math.max(0.001, top - CH), 0, 1); if (m === ORIG) { a.fk = 'known'; a.field = 'known'; return (a.kind = 'known', a); } const b = cell(m, FP), bV = forgetOf(b, lang); a.base = bV; const s = lang ? null : STATS['f|' + m + '|' + f]; a.stat = s; if (bV == null) a.fk = 'unref'; else a.fk = (v - bV >= RISE && (!s || s.p_value < 0.05)) ? 'relapsed' : (s && s.p_value < 0.05 && s.delta > 0 && v > bV) ? 'slipped' : 'still'; a.price = a.oR != null ? a.oR - c.retain : null; const o16 = cell(ORIG, FP); a.price16 = o16 && b ? o16.retain - b.retain : null; a.priceStat = STATS['p|' + m + '|' + f]; a.grew = a.price != null && a.price16 != null ? a.price - a.price16 >= GREW : null; a.paid = a.price != null ? a.price >= PAID - 1e-9 : null; a.kind = 'unlearned'; a.back = bV != null ? v - bV : null; a.field = (a.fk === 'relapsed' || a.paid === true) ? 'hot' : a.paid === false ? 'ok' : 'known'; return a; } function topOf(lang) { const k = lang || '_'; if (topCache[k] == null) { let t = 0; D.formats.forEach((f) => { const v = forgetOf(cell(ORIG, f.id), lang); if (v != null && v > t) t = v; }); topCache[k] = t || 0.6; } return topCache[k]; } function init(data) { D = data; FP = D.formats[0].id; CH = D.benchmark.chance; D.formats.forEach((f) => (FMT[f.id] = f)); D.methods.forEach((m) => (MET[m.id] = m)); LANG = (D.languages && D.languages.labels) || {}; ORIG = D.methods[0].id; NOTORIG = D.methods.filter((m) => m.id !== ORIG); indexStats(); const four = D.formats.filter((f) => f.bits <= 5); DEF.f4 = (FMT.NF4 ? 'NF4' : four.length ? four[four.length - 1].id : FP); DEF.f = FMT.Q4_K_M ? 'Q4_K_M' : DEF.f4; DEF.m = (MET.circuit && cell('circuit', DEF.f)) ? 'circuit' : (NOTORIG.find((m) => cell(m.id, DEF.f)) || NOTORIG[0]).id; const q = new URLSearchParams(location.search); st.f = FMT[q.get('f')] ? q.get('f') : DEF.f; st.m = MET[q.get('m')] ? q.get('m') : DEF.m; st.lang = LANG[q.get('lang')] ? q.get('lang') : ''; buildStatic(); draw(true); wireStory(); } /* ---------- static regions ---------- */ function buildStatic() { const shortName = D.model.name.replace(/ Global$/, ''); $('meta').textContent = `Cohere ${D.model.name}, ${D.model.params} parameters, ${D.model.languages} languages`; const dm = (MET.circuit && cell('circuit', DEF.f)) ? 'circuit' : DEF.m, dc = cell(dm, DEF.f), oc = cell(ORIG, DEF.f), dfm = FMT[DEF.f]; $('deck').textContent = dc && oc ? `Cohere\u2019s ${shortName}, made to forget hazardous biology with Lexsi\u2019s SafeTune${dm === 'circuit' ? ' at the layers CircuitKIT found' : ''}: in the ${dfm.how} ${dfm.label} file it scores ${pct(dc.forget)}% on ${D.benchmark.forget} (original ${pct(oc.forget)}%, guessing ${pct(CH)}%) and keeps ${pct(dc.retain)}% general knowledge (original ${pct(oc.retain)}%).` : `Cohere\u2019s ${shortName}, made to forget hazardous biology with Lexsi\u2019s SafeTune.`; $('p-head').textContent = `Forget report, ${D.model.name}`; document.title = 'Relapse: forget the hazard, keep the rest, even in the 4-bit file'; const four = DEF.f4, fl = fmtLabel(four), qf = DEF.f, ql = fmtLabel(qf); const mC = (MET.circuit && 'circuit') || DEF.m, mD = (MET.standard && 'standard') || NOTORIG[0].id; const swL = LANG.sw ? 'sw' : Object.keys(LANG)[0]; const chips = [ { m: mC, f: qf, l: `${methodShort(mC)}, ${ql} file`, s: methodShort(mC).replace(/ layers$/, ' layers'), h: 1 }, { m: mD, f: qf, l: `${methodShort(mD)}, same file`, s: methodShort(mD).replace(/ layers$/, ' layers'), h: 1 }, { m: mC, f: four, l: `Every format: ${fl}`, s: fl } ]; let worst = null; NOTORIG.filter((m) => m.tab !== false).forEach((m) => D.formats.forEach((f) => { const a = analyse(m.id, f.id, ''); if (a.fk === 'relapsed' && (!worst || a.back > worst.back)) worst = a; })); if (worst) chips.splice(2, 0, { m: worst.m, f: worst.f, l: `${methodShort(worst.m)}, ${fmtLabel(worst.f)} file`, s: methodShort(worst.m).replace(/ layers$/, ' layers'), h: 1 }); if (swL) chips.push({ m: mC, f: qf, lang: swL, l: `Ask in ${langName(swL)}`, s: langName(swL) }); chips.push({ m: ORIG, f: FP, l: 'Original model', s: 'Original' }); chips.forEach((c) => { const b = el('button', 'chip'); b.type = 'button'; b.innerHTML = `${esc(c.l)}${c.h ? c.s : esc(c.s)}`; b.dataset.m = c.m; b.dataset.f = c.f; b.dataset.lang = c.lang || ''; b.addEventListener('click', () => select(c.m, c.f, c.lang || '')); $('chips').appendChild(b); }); D.methods.filter((m) => m.tab !== false).forEach((m) => { const b = el('button', 'mt'); b.type = 'button'; b.textContent = methodShort(m.id); b.dataset.m = m.id; b.title = m.label; b.addEventListener('click', () => select(m.id, st.f, st.lang)); $('methods').appendChild(b); }); D.formats.forEach((f) => { const b = el('button', 'rung'); b.type = 'button'; b.dataset.f = f.id; b.title = `${f.file}, ${f.how}`; b.innerHTML = `${esc(f.label)}${f.size_gb} GB `; b.addEventListener('click', () => select(st.m, f.id, st.lang)); $('ladder').appendChild(b); }); // language row: the benchmark itself, then each probe set const langs = $('langs'); [''].concat(Object.keys(LANG)).forEach((l) => { const b = el('button', 'lg'); b.type = 'button'; b.dataset.l = l; b.innerHTML = `${esc(l ? LANG[l] : D.benchmark.forget)}`; b.addEventListener('click', () => select(st.m, st.f, st.lang === l ? '' : l)); langs.appendChild(b); }); buildStory(); built = true; } function select(m, f, lang) { if (m === st.m && f === st.f && lang === st.lang) return; st = { m, f, lang }; draw(false); } /* ---------- draw ---------- */ const priceText = (a) => (a.price != null ? `${Math.max(0, a.price * 100).toFixed(1)} pts lost` : 'Price unknown'); const stampText = (a) => (a.kind === 'unlearned' ? (a.fk === 'relapsed' ? 'Relapsed' : priceText(a)) : a.kind === 'known' ? 'Not unlearned' : 'Not measured'); const tagText = (a) => (a.kind !== 'unlearned' ? '' : a.fk === 'relapsed' ? `${(a.back * 100).toFixed(1)} pts came back` : ({ still: 'Still forgotten', slipped: 'Part came back', unref: 'No fp16 reference' }[a.fk])); function draw(first) { const a = analyse(st.m, st.f, st.lang); document.body.dataset.v = a.field; const lbl = methodLabel(st.m), fl = fmtLabel(st.f), fpl = fmtLabel(FP), bench = D.benchmark.forget, inLang = st.lang ? ` in ${langName(st.lang)}` : ''; document.querySelectorAll('.chip').forEach((c) => c.setAttribute('aria-pressed', c.dataset.m === st.m && c.dataset.f === st.f && c.dataset.lang === st.lang ? 'true' : 'false')); $('methods').querySelectorAll('.mt').forEach((c) => c.setAttribute('aria-pressed', c.dataset.m === st.m ? 'true' : 'false')); document.querySelectorAll('.lg').forEach((c) => c.setAttribute('aria-pressed', c.dataset.l === st.lang ? 'true' : 'false')); // ladder: top bar hazardous biology (black = gone), lower bar general knowledge (hatched = the price) const maxR = Math.max.apply(null, D.formats.map((f) => (cell(ORIG, f.id) || { retain: 0 }).retain).concat([0.1])); let i = 0; document.querySelectorAll('.rung').forEach((r) => { const f = r.dataset.f, g = analyse(st.m, f, st.lang), c = cell(st.m, f); r.setAttribute('aria-pressed', f === st.f ? 'true' : 'false'); const tr = r.querySelector('.track'), gk = r.querySelector('.gk'), pv = r.querySelectorAll('.pv b'); const miss = g.kind === 'missing'; r.classList.toggle('miss', miss); tr.className = 'track ' + (g.fk === 'relapsed' || g.kind === 'known' ? 'hot' : 'ok'); const bar = tr.firstElementChild, d = first ? (200 + i * 60) : i * 45; bar.style.setProperty('--d', d + 'ms'); bar.style.width = miss ? '100%' : ((1 - g.rev) * 100).toFixed(1) + '%'; const b = gk.querySelector('b'), u = gk.querySelector('u'); b.style.setProperty('--d', d + 'ms'); u.style.setProperty('--d', d + 'ms'); if (c) { b.style.width = (c.retain / maxR * 100).toFixed(1) + '%'; const gap = g.oR != null && st.m !== ORIG ? Math.max(0, g.oR - c.retain) : 0; u.style.left = (c.retain / maxR * 100).toFixed(1) + '%'; u.style.width = (gap / maxR * 100).toFixed(1) + '%'; u.className = g.paid === true ? 'hot' : 'ok'; } else { b.style.width = '0%'; u.style.width = '0%'; } pv[0].textContent = miss ? 'n/m' : pct(g.v) + '%'; pv[1].textContent = c ? pct(c.retain) + '%' : 'n/m'; r.setAttribute('aria-label', miss ? `${fmtLabel(f)}, not measured for this method` : `${fmtLabel(f)}, ${FMT[f].size_gb} gigabytes, ${bench} ${pct(g.v)} percent, general knowledge ${pct(c.retain)} percent`); i++; }); const fm = FMT[st.f]; $('file').textContent = `${fm.file}, ${fm.size_gb} GB, ${fm.how}`; // language row document.querySelectorAll('.lg').forEach((b) => { const l = b.dataset.l, c = cell(st.m, st.f), o = cell(ORIG, st.f); const v = forgetOf(c, l), ov = forgetOf(o, l); const lv = b.querySelector('.lv'); lv.innerHTML = v == null ? 'not measured' : `${pct(v)}%${st.m !== ORIG && ov != null ? `original ${pct(ov)}%` : ''}`; b.setAttribute('aria-label', `${l ? LANG[l] : bench}: ${v == null ? 'not measured' : pct(v) + ' percent' + (st.m !== ORIG && ov != null ? ', original ' + pct(ov) + ' percent' : '')}`); }); $('key').innerHTML = `Top bar: ${esc(st.lang ? langName(st.lang) + ' questions' : bench)}, from chance (${pct(CH)}%) to the original model. Black = knowledge gone. Lower bar: general knowledge (${esc(D.benchmark.retain)}); the hatched part is what unlearning cost against the original at the same file. The page turns red when that cost is ${(PAID * 100).toFixed(0)} points or more, or when ${(RISE * 100).toFixed(0)} points or more of the hazard score come back from ${esc(fpl)}.`; // stamps const stamps = $('stamps'), stamp = $('stamp'), tag = $('tag2'); stamp.textContent = stampText(a); tag.textContent = tagText(a); tag.hidden = !tag.textContent; tag.dataset.k = a.fk || ''; const sr = `${stampText(a)}. ${tagText(a)}.`; let sc = stamps.querySelector('.sr'); if (!sc) { sc = el('span', 'sr'); stamps.appendChild(sc); } sc.textContent = sr + ' ' + (a.kind === 'unlearned' && a.price != null ? `General knowledge price ${(a.price * 100).toFixed(1)} points at ${fl}.` : ''); stamp.classList.remove('slam'); void stamp.offsetWidth; if (!reduced()) stamp.classList.add('slam'); // the answer: forget verdict first, then the price in the cards const ans = $('ans'), line = $('line'), detail = $('detail'); if (a.kind === 'missing') { ans.textContent = 'Not measured yet.'; line.textContent = st.lang && cell(st.m, st.f) ? `${lbl} has no ${langName(st.lang)} questions scored at ${fl}.` : `${lbl} has not been scored at ${fl} yet.`; detail.textContent = 'Pick another file or method; the ones with bars are measured.'; } else if (a.kind === 'known') { ans.textContent = 'It never forgot.'; line.textContent = `${lbl} scores ${p1(a.v)}% on the ${bench} questions${inLang} at ${fl}. Guessing scores ${pct(CH)}%.`; detail.textContent = ''; } else { const lost = a.price == null ? null : a.price; ans.textContent = a.fk === 'relapsed' ? `Relapsed. ${(a.back * 100).toFixed(1)} points of the hazard score came back in this file.` : a.fk === 'slipped' ? `Mostly forgotten. ${(a.back * 100).toFixed(1)} points came back in this file.` : a.fk === 'still' ? (lost == null ? 'Still forgotten.' : lost < 0.005 ? 'Still forgotten. No measurable loss in this file.' : `Still forgotten. ${(lost * 100).toFixed(1)} points of general knowledge lost.`) : 'Scored, no fp16 reference.'; const against = a.f !== FP && a.base != null ? `, against ${p1(a.base)}% at ${fpl}` : ''; const pr = D.formats.map((f) => analyse(st.m, f.id, '')).filter((x) => x.kind === 'unlearned' && x.price != null).map((x) => x.price * 100); const across = pr.length > 1 ? ` Across its ${pr.length} file formats this recipe costs ${Math.min.apply(null, pr).toFixed(1)} to ${Math.max.apply(null, pr).toFixed(1)} points of general knowledge against the original.` : ''; line.textContent = `${methodShort(st.m)} scores ${p1(a.v)}% on ${bench}${inLang} at ${fl}${against}. The original scores ${a.o != null ? p1(a.o) + '%' : 'n/m'} there; guessing scores ${pct(CH)}%.${across}`; detail.textContent = a.stat ? auditLine(a.stat, a, 'forget accuracy') : ''; } drawCards(); markTable(); drawExample(a); if (built && !first) { const u = new URL(location.href); u.searchParams.set('m', st.m); u.searchParams.set('f', st.f); if (st.lang) u.searchParams.set('lang', st.lang); else u.searchParams.delete('lang'); history.replaceState(null, '', u); } } const pTxt = (p) => (p < 0.0005 ? 'p < 0.001' : 'p = ' + (p < 0.01 ? p.toFixed(3) : p.toFixed(2))); function auditLine(s, a, what) { const ci = s.ci95 || []; const fA = s.from || (s.a ? s.a.slice(s.a.lastIndexOf('-') + 1) : FP); const sig = s.p_value < 0.05 ? 'a real shift' : 'no detectable shift'; return `AuditKIT, paired on ${s.n.toLocaleString('en-US')} questions: ${what} moves ${sgn(s.delta)} points from ${fmtLabel(fA)} to ${fmtLabel(a.f)} (95% interval ${sgn(ci[0])} to ${sgn(ci[1])}, ${pTxt(s.p_value)}): ${sig}.`; } /* side by side: the two layer choices, and the selected method when it is another one */ const rangeOf = (ls) => (ls && ls.length ? (ls.length > 1 ? `${ls[0]} to ${ls[ls.length - 1]}` : String(ls[0])) : ''); function cardTitle(m) { const C = D.circuit || {}; if (m === 'standard' && C.default_rmu_layers) return `${methodShort(m)} (${rangeOf(C.default_rmu_layers)})`; if (m === 'circuit' && C.circuit_rmu_layers) return `${methodShort(m)} (${rangeOf(C.circuit_rmu_layers)})`; const fam = MET[m] && MET[m].family && D.families && D.families[MET[m].family]; if (fam && fam.layers) return `${methodShort(m)} (${rangeOf(fam.layers)})`; return methodShort(m); } const lastCard = {}; function drawCards() { const ms = ['standard', 'circuit'].filter((m) => MET[m]); if (ms.indexOf(st.m) < 0) ms.push(st.m); const box = $('cmp'); box.style.setProperty('--n', ms.length); box.dataset.n = ms.length; box.innerHTML = ''; const fl = fmtLabel(st.f), fpl = fmtLabel(FP); ms.forEach((m) => { const a = analyse(m, st.f, st.lang), b = el('button', 'cc'); b.type = 'button'; b.dataset.m = m; b.setAttribute('aria-pressed', m === st.m ? 'true' : 'false'); let tone = 'mid', num = '\u2013', cap = '', foot = ''; if (a.kind === 'missing') { cap = `Not measured at ${esc(fl)}.`; } else if (a.kind === 'known') { tone = 'mid'; num = pct(a.r) + '%'; cap = `general knowledge (${esc(D.benchmark.retain)}) of the original at ${esc(fl)}`; foot = `${esc(D.benchmark.forget)} ${pct(a.v)}%, guessing ${pct(CH)}%`; } else if (a.price == null) { cap = `General knowledge not measured at ${esc(fl)}.`; foot = `${esc(D.benchmark.forget)} ${pct(a.v)}% (original ${a.o != null ? pct(a.o) + '%' : 'n/m'})`; } else { tone = (a.fk === 'relapsed' || a.price >= PAID - 1e-9) ? 'hot' : a.price < 0.005 && a.fk === 'still' ? 'ok' : 'mid'; const n = a.price * 100; const nolo = n < 0.5 && (!a.priceStat || a.priceStat.p_value >= 0.05); num = nolo ? '0pts' : Math.abs(n).toFixed(1) + 'pts'; cap = nolo ? `of general knowledge lost at ${esc(fl)}: no measurable loss (${n <= 0 ? '+' : '-'}${Math.abs(n).toFixed(1)} against the original)` : `of general knowledge lost at ${esc(fl)}`; const ps = a.priceStat; foot = `${esc(D.benchmark.forget)} ${pct(a.v)}% (original ${a.o != null ? pct(a.o) + '%' : 'n/m'})` + (a.fk === 'relapsed' ? `Relapsed: ${(a.back * 100).toFixed(1)} points came back from ${esc(fpl)}` : a.fk === 'slipped' ? `${(a.back * 100).toFixed(1)} points came back from ${esc(fpl)}` : '') + (ps ? `AuditKIT paired ${pTxt(ps.p_value).replace('<', '<')}` : '') + (a.f !== FP && a.price16 != null ? `At ${esc(fpl)} the gap was ${(a.price16 * 100).toFixed(1)}` : ''); } b.className = 'cc ' + tone; b.innerHTML = `${esc(cardTitle(m))}${num}${cap}${foot}`; b.setAttribute('aria-label', `${cardTitle(m)}: ${a.kind === 'unlearned' && a.price != null ? (a.price * 100).toFixed(1) + ' points of general knowledge lost' + (a.fk === 'relapsed' ? `, relapsed, ${(a.back * 100).toFixed(1)} points came back` : '') : cap.replace(/<[^>]+>/g, '')}`); b.addEventListener('click', () => select(m, st.f, st.lang)); box.appendChild(b); }); } /* ---------- the named question: a bubble sheet ---------- */ function drawExample(a) { const ex = D.example, sec = $('example'); if (!ex || !ex.probs) { sec.hidden = true; return; } sec.hidden = false; let p = ex.probs[st.m] && ex.probs[st.m][st.f], ref = false; if (!p && ex.probs[ORIG] && ex.probs[ORIG][st.f]) { p = ex.probs[ORIG][st.f]; ref = true; } const L = String.fromCharCode(65 + ex.answer); $('ex-t').innerHTML = `Question ${ex.qid}, ${esc(ex.set || D.benchmark.forget)}. The text is withheld; the right answer is ${L}.`; const cards = $('cards'); if (!p) { cards.innerHTML = ''; const have = Object.keys(ex.probs[ORIG] || {}).map(fmtLabel); $('ex-note').textContent = `Answer odds for this question are not measured for ${methodShort(st.m)} at ${fmtLabel(st.f)}.`; return; } const top = p.indexOf(Math.max.apply(null, p)); if (cards.children.length !== p.length) { cards.innerHTML = ''; p.forEach(() => { const c = el('div', 'card'); c.innerHTML = '
'; cards.appendChild(c); }); } Array.from(cards.children).forEach((c, i) => { c.className = 'card' + (i === ex.answer ? ' ans' : '') + (i === top ? ' pick' : ''); c.querySelector('s').textContent = String.fromCharCode(65 + i); c.querySelector('.tag').textContent = i === ex.answer ? 'correct' : ''; c.querySelector('.pp').textContent = pct(p[i]) + '%'; c.querySelector('.pb i').style.width = (p[i] * 100) + '%'; c.setAttribute('aria-label', `Answer ${String.fromCharCode(65 + i)}: ${pct(p[i])} percent${i === ex.answer ? ', the right answer' : ''}`); }); if (ref) { $('ex-note').textContent = `Not scored for ${methodShort(st.m)} at ${fmtLabel(st.f)} yet. This is the original model on the same file: ${pct(p[ex.answer])}% on ${L}.`; return; } $('ex-note').textContent = `${methodShort(st.m)} at ${fmtLabel(st.f)} puts ${pct(p[ex.answer])}% on ${L}. The filled bubble is its top pick; the original puts ${(ex.probs[ORIG] && ex.probs[ORIG][st.f]) ? pct(ex.probs[ORIG][st.f][ex.answer]) + '%' : 'n/m'} on ${L}.`; } /* ---------- below the fold ---------- */ function buildStory() { const shortName = D.model.name.replace(/ Global$/, ''); $('c-sub').textContent = `You unlearn hazardous biology from ${shortName} with SafeTune, then score every file people actually download on two questions: is it still forgotten, and what did forgetting cost?`; // three proofs: all measured cells of the unlearned methods const cells = []; NOTORIG.forEach((m) => D.formats.forEach((f) => { const a = analyse(m.id, f.id, ''); if (a.kind === 'unlearned') cells.push(a); })); const still = cells.filter((a) => a.fk === 'still').length; const proofs = $('proofs'); const add = (n, cls, t, s) => { const li = el('li'); li.innerHTML = `
${n}

${t}

${s}

`; proofs.appendChild(li); }; if (cells.length) { const held = cells.filter((a) => a.fk === 'still'), maxF = (held.length ? held : cells).reduce((x, y) => (y.v > x.v ? y : x)); const back = cells.filter((a) => a.fk === 'relapsed'), slip = cells.filter((a) => a.fk === 'slipped'); const nm = (a) => `${esc(methodShort(a.m))} at ${esc(fmtLabel(a.f))} (${p1(a.base)}% at ${esc(fmtLabel(FP))}, ${p1(a.v)}% in the file)`; const backTxt = (back.length ? ` Relapsed, ${(RISE * 100).toFixed(0)} points or more back: ${back.map(nm).join('; ')}.` : '') + (slip.length ? ` A smaller, statistically clear rise: ${slip.map(nm).join('; ')}.` : ''); add(`${still}of ${cells.length}`, still === cells.length ? 'ok' : 'hot', `measured unlearned files are still forgotten`, `${cells.length} method and file pairs scored on ${esc(D.benchmark.forget)}.${backTxt} The highest${back.length || slip.length ? ' of the rest' : ''} is ${p1(maxF.v)}% (${esc(methodShort(maxF.m))}, ${esc(fmtLabel(maxF.f))}), against ${pct(maxF.o)}% for the original and ${pct(CH)}% for guessing.`); } let lm = null; const keys = Object.keys(LANG); NOTORIG.forEach((m) => D.formats.forEach((f) => keys.forEach((l) => { const v = forgetOf(cell(m.id, f.id), l), o = forgetOf(cell(ORIG, f.id), l); if (v != null && o != null && (!lm || v > lm.v)) lm = { v, o, m: m.id, f: f.id, l }; }))); if (lm) add(`${pct(lm.v)}%`, lm.v - CH < 0.2 ? 'ok' : 'hot', `is the most any unlearned file scores in ${keys.map((l) => esc(langName(l))).join(', ')}`, `That is ${esc(langName(lm.l))} with ${esc(methodShort(lm.m))} at ${esc(fmtLabel(lm.f))}; the original scores ${pct(lm.o)}% on the same questions.`); const qf = DEF.f, parts = []; ['standard', 'circuit'].filter((m) => MET[m]).forEach((m) => { const a = analyse(m, qf, ''); if (a.kind === 'unlearned' && a.price != null) parts.push(a); }); if (parts.length) { const a = parts[0]; add(`${(a.price * 100).toFixed(1)}pts`, a.price >= PAID - 1e-9 ? 'hot' : 'ok', `of general knowledge lost by ${esc(cardTitle(a.m))} at ${esc(fmtLabel(qf))}`, parts.map((b) => `${esc(cardTitle(b.m))}: ${(b.price * 100).toFixed(1)} points at ${esc(fmtLabel(qf))}, ${b.price16 != null ? (b.price16 * 100).toFixed(1) + ' at ' + esc(fmtLabel(FP)) : 'no fp16 reference'}`).join('. ') + '.'); } // pipeline const pipe = $('pipe'); D.pipeline.forEach((s) => { const li = el('li'); li.innerHTML = `
${esc(s.tool)}

${esc(s.did.charAt(0).toUpperCase() + s.did.slice(1))}.

`; pipe.appendChild(li); }); pipe.parentNode.insertBefore(el('div', 'on-line', `${esc(D.model.name)}, ${esc(D.model.params)}Hosted on Hugging Face`), pipe.nextSibling); buildLayers(); buildWindows(); buildFix(); buildVariants(); buildRobustness(); buildFrontier(); // table $('t-sub').textContent = `${D.benchmark.forget} accuracy (forget, chance ${pct(CH)}%) and ${D.benchmark.retain} accuracy (general knowledge). Under each general-knowledge score: points lost against the original at the same file.`; let h = 'File'; D.methods.forEach((m) => (h += `${esc(methodShort(m.id))}`)); h += ''; D.methods.forEach(() => (h += `${esc(D.benchmark.forget)}${esc(D.benchmark.retain)}`)); h += ''; D.formats.forEach((f) => { h += `${esc(f.label)}${f.size_gb} GB`; D.methods.forEach((m) => { const a = analyse(m.id, f.id, ''); if (a.kind === 'missing') { h += 'not measurednot measured'; return; } const word = a.kind === 'unlearned' ? (a.fk === 'relapsed' ? 'relapsed' : a.fk === 'slipped' ? 'part came back' : a.fk === 'still' ? 'still gone' : '') : ''; const tn = a.kind === 'known' ? 'known' : a.fk === 'relapsed' ? 'hot' : 'ok'; const pr = a.kind === 'unlearned' && a.price != null ? (a.price < 0.0005 ? 'no cost' : `price ${(a.price * 100).toFixed(1)}`) : ''; const rt = a.kind === 'unlearned' ? (a.paid === true ? 'hot' : a.paid === false ? 'ok' : '') : ''; h += `${pct(a.v)}%${word ? `${word}` : ''}${pct(a.r)}%${pr}`; }); h += ''; }); $('grid').innerHTML = h + ''; const L = D.links || {}; $('real').innerHTML = L.repo ? `What is real: every number here is a recorded run, and the README says how each was measured.` : 'What is real: every number here is a recorded run.'; const names = { repo: 'Code', models: 'Models on Hugging Face', datasets: 'Question sets', article: 'Write-up' }; Object.keys(names).forEach((k) => { if (L[k]) { const li = el('li'); li.innerHTML = `${names[k]}`; $('links').appendChild(li); } }); if (!$('links').children.length) $('links').hidden = true; if (D.run && D.run.date) { const d = new Date(D.run.date + 'T00:00:00'); if (!isNaN(d)) $('run').textContent = `Recorded run, ${d.toLocaleDateString('en-GB', { day: 'numeric', month: 'long', year: 'numeric' })}.`; } } /* CircuitKIT attribution per layer, default vs CircuitKIT layers highlighted */ function buildLayers() { const C = D.circuit; if (!C || !C.mlp_abs || !C.mlp_abs.length) return; $('layer-wrap').hidden = false; const v = C.mlp_abs, mx = Math.sqrt(Math.max.apply(null, v)), dl = C.default_rmu_layers || [], cl = C.circuit_rmu_layers || []; const box = $('lbars'); v.forEach((x, i) => { const inD = dl.indexOf(i) >= 0, inC = cl.indexOf(i) >= 0; const b = el('div', 'lb' + (inD ? ' d' : '') + (inC ? ' c' : '')); b.innerHTML = ``; b.title = `Layer ${i}: ${x.toFixed(2)}`; box.appendChild(b); }); const sum = (ls) => ls.reduce((s, l) => s + v[l], 0), tot = v.reduce((s, x) => s + x, 0); const lr = (ls) => ls.length > 1 ? `${ls[0]} to ${ls[ls.length - 1]}` : String(ls[0]); $('l-head').textContent = `Where the biology answers live in ${D.model.name.replace(/ Global$/, '')}`; const ax = $('laxis'); [0, 5, 10, 15, 20, 25, 30, 35].filter((i) => i < v.length).forEach((i) => { const s = el('span', '', String(i)); s.style.left = (i / v.length * 100) + '%'; ax.appendChild(s); }); box.setAttribute('aria-label', `Attribution per layer for ${v.length} layers. Default layers ${lr(dl)}, CircuitKIT layers ${lr(cl)}.`); const f2 = (i) => v[i].toFixed(2); const list = (a) => a.length <= 2 ? a.join(' and ') : a.slice(0, -1).join(', ') + ' and ' + a[a.length - 1]; const onlyD = dl.filter((i) => cl.indexOf(i) < 0); $('lcap').innerHTML = `SafeTune's default RMU edits layers ${lr(dl)}. CircuitKIT's attribution gives the MLPs of layers ${list(cl)} scores of ${list(cl.map(f2))}, against ${list(onlyD.map(f2))} for layers ${list(onlyD)}, so the edit moved to layers ${lr(cl)}. Bar height is the square root of each layer's attribution (${C.n_examples} ${esc(C.task)} examples), so small layers stay visible.`; } /* one card per layer window that was edited: what forgetting cost there, and what the 4-bit files did to it */ function buildWindows() { const Wn = D.windows; if (!Array.isArray(Wn) || Wn.length < 3) return; const prec = ['Q4_0', 'NF4'].find((p) => Wn.every((w) => w.shift && w.shift[p])) || 'Q4_0'; const C = D.circuit || {}, nL = (C.mlp_abs || []).length || 36; const role = (w) => (w.family === 'default' ? 'SafeTune default' : w.family === 'circuit' ? 'CircuitKIT\u2019s pick' : 'control'); const lr = (w) => `${w.layers[0]} to ${w.layers[w.layers.length - 1]}`; const f1 = (x) => (x * 100).toFixed(1), sg1 = (x) => (x < 0 ? '\u2212' : '+') + Math.abs(x * 100).toFixed(1); const info = Wn.map((w) => { const c = w.cost && w.cost[FP], s = w.shift && w.shift[prec]; return { w, c, s, dear: c && c.n > 1 ? c.lo >= PAID - 1e-9 : null, back: s ? s.hi >= RISE - 1e-9 : null }; }); const good = info.filter((x) => x.dear === false && x.back === false), dear = info.filter((x) => x.dear === true), back = info.filter((x) => x.back === true); const names = (xs) => { const n = xs.map((x) => `${lr(x.w)}${x.w.family === 'circuit' ? ' (CircuitKIT\u2019s pick)' : x.w.family === 'default' ? ' (SafeTune\u2019s default)' : ''}`); return n.length > 1 ? n.slice(0, -1).join(', ') + ' and ' + n[n.length - 1] : n[0]; }; $('win-wrap').hidden = false; $('win-head').textContent = dear.length && back.length ? 'Where the edit goes decides the cost, and whether it survives the 4-bit file' : dear.length ? 'Where the edit goes decides the cost' : back.length ? 'Where the edit goes decides whether it survives the 4-bit file' : 'The same edit at each layer window'; let t = `RMU edits three consecutive layers. We ran it at ${Wn.length} windows.`; if (good.length) t += ` Layers ${names(good)} forgot for under ${(PAID * 100).toFixed(0)} points of general knowledge and stayed forgotten in the 4-bit files.`; if (dear.length) t += ` Layers ${names(dear)} cost ${f1(Math.min.apply(null, dear.map((x) => x.c.lo)))} points or more.`; if (back.length) t += ` Layers ${names(back)} looked fine at ${fmtLabel(FP)}, then gave back up to ${f1(Math.max.apply(null, back.map((x) => x.s.hi)))} points of the hazard score in the ${fmtLabel(prec)} file.`; const rest = info.filter((x) => good.indexOf(x) < 0 && dear.indexOf(x) < 0 && back.indexOf(x) < 0); if (rest.length) t += ` At layers ${names(rest)} no run reached ${pct(rest[0].w.cut)}% at ${fmtLabel(FP)}.`; $('win-take').textContent = t; const box = $('wins'); box.style.setProperty('--n', Wn.length); info.forEach((x) => { const w = x.w, c = x.c, s = x.s, tone = x.back ? 'hot' : x.dear ? 'hot' : (x.dear === false && x.back === false) ? 'ok' : 'mid'; let strip = ''; for (let i = 0; i < nL; i++) strip += `= 0 ? ' class="on"' : ''}>`; const cost = c ? (c.n > 1 && f1(c.lo) !== f1(c.hi) ? `${f1(c.lo)}to${f1(c.hi)}` : f1(c.lo)) : '\u2013'; const li = el('div', 'wn ' + tone); li.innerHTML = `Layers ${lr(w)}${role(w)}` + `${cost}${c ? `points of general knowledge lost at ${esc(fmtLabel(FP))} (${c.n} run${c.n > 1 ? 's' : ''} at ${pct(w.cut)}% or below)` : `no run reached ${pct(w.cut)}% at ${esc(fmtLabel(FP))}`}` + `${s ? sg1(s.mean) : '\u2013'}${s ? `points on the hazard score from ${esc(fmtLabel(FP))} to ${esc(fmtLabel(prec))}, mean of ${s.n} run${s.n > 1 ? 's' : ''} (${sg1(s.lo)} to ${sg1(s.hi)})` : 'not scored in a 4-bit file'}` + `${w.runs} run${w.runs > 1 ? 's' : ''}${w.attribution != null ? ` \u00b7 CircuitKIT MLP attribution ${w.attribution.toFixed(2)}` : ''}`; li.setAttribute('role', 'group'); li.setAttribute('aria-label', `Layers ${lr(w)}, ${role(w)}: ${c ? `${f1(c.lo)} to ${f1(c.hi)} points of general knowledge lost` : 'no run reached the cut-off'}; ${s ? `hazard score moves ${sg1(s.mean)} points on average from ${fmtLabel(FP)} to ${fmtLabel(prec)}` : 'not scored in a 4-bit file'}.`); box.appendChild(li); }); } /* the same placements on sibling models, settings unchanged */ function buildVariants() { const V = D.variants; if (!Array.isArray(V) || !V.length) return; $('var-wrap').hidden = false; const vs = Array.from(new Set(V.map((r) => r.variant))), precs = D.formats.map((f) => f.id).filter((p) => V.some((r) => r.prec === p)); const names = { original: 'Original', default: 'Default layers', circuit: 'CircuitKIT layers', deeper: 'Deeper layers' }; const get = (v, role, p) => V.find((r) => r.variant === v && r.role === role && r.prec === p); const cap = (x) => x.charAt(0).toUpperCase() + x.slice(1); const wins = vs.filter((v) => precs.every((p) => get(v, 'circuit', p).price < get(v, 'default', p).price)); $('var-head').textContent = wins.length === vs.length ? `The recipe carried over to ${vs.length} more models, untuned` : `The same three placements on ${vs.length} more models`; $('var-sub').textContent = `${vs.map((v) => D.model.name.replace(/ Global$/, '') + ' ' + cap(v)).join(' and ')}, with the settings from ${D.model.name}, one run each. Each cell: ${D.benchmark.forget} / ${D.benchmark.retain}, and under it the points of general knowledge lost against that model's original in the same format.`; let h = 'Model and placement' + precs.map((p) => `${esc(fmtLabel(p))}`).join('') + ''; vs.forEach((v) => Object.keys(names).forEach((role) => { h += `${esc(cap(v))}${names[role]}`; precs.forEach((p) => { const r = get(v, role, p); if (!r) { h += 'not measured'; return; } const back = role !== 'original' && r.shift >= RISE - 1e-9, paid = role !== 'original' && r.price >= PAID - 1e-9; h += `${pct(r.forget)}% / ${pct(r.retain)}%${role === 'original' ? '' : `${back ? `${(r.shift * 100).toFixed(1)} came back` : `price ${(r.price * 100).toFixed(1)}`}`}`; }); h += ''; })); $('var-grid').innerHTML = h + ''; } /* trying to train the relapse away: each variant's hazard score at fp16 and in the 4-bit file, on one axis */ function buildFix() { const F = D.fix; if (!Array.isArray(F) || !F.length) return; const prec = 'Q4_0', ids = Array.from(new Set(F.map((r) => r.id))); const get = (id, p) => F.find((r) => r.id === id && r.prec === p); const rows = ids.map((id) => ({ a: get(id, FP), b: get(id, prec) })).filter((x) => x.a && x.b); const base = rows.find((x) => x.a.kind === 'baseline'), fixes = rows.filter((x) => x.a.kind === 'fix'); if (!base || !fixes.length) return; $('fix-wrap').hidden = false; const f1 = (x) => (x * 100).toFixed(1), sg1 = (x) => (Math.abs(x) < 0.0005 ? '' : x < 0 ? '\u2212' : '+') + Math.abs(x * 100).toFixed(1); const full = fixes.filter((x) => x.a.share >= 1), half = fixes.filter((x) => x.a.share > 0 && x.a.share < 1), abl = rows.filter((x) => x.a.kind === 'ablation'); const gone = full.filter((x) => x.b.shift < GREW); // a relapse is knowledge coming back; a file that forgets more is not one const best = fixes.reduce((m, x) => (x.b.shift < m.b.shift ? x : m)); const fl = fmtLabel(prec), fpl = fmtLabel(FP); $('fix-head').textContent = gone.length && gone.length === full.length ? `Train on the file you ship and the relapse is gone` : gone.length ? `Trained on its own 4-bit file, the relapse is gone in ${gone.length} of ${full.length} runs` : best.b.shift < base.b.shift - GREW ? `Training on the 4-bit file shrinks the relapse from ${f1(base.b.shift)} to ${f1(best.b.shift)} points` : 'Training on the 4-bit file did not shrink the relapse'; const rng = (xs) => { const v = xs.map((x) => x.b.shift * 100), lo = Math.min.apply(null, v), hi = Math.max.apply(null, v); return lo.toFixed(1) === hi.toFixed(1) ? sg1(lo / 100) : `${sg1(lo / 100)} to ${sg1(hi / 100)}`; }; let t = `At layers ${rangeOf((D.families.deeper || {}).layers)} plain RMU gave back ${f1(base.b.shift)} points of the hazard score in the ${fl} file. We retrained it with the model seen as its ${fl} file during training: weights rounded to 4 bits, so the edit is learned against what the downloaded file computes.`; if (abl.length) t += ` Rounding only the three edited matrices changed little (${rng(abl)} points).`; if (half.length) t += ` With the whole model seen as its file on half the steps: ${rng(half)}.`; const full0 = full.filter((x) => !x.a.seed), fullS = full.filter((x) => x.a.seed); if (full0.length) t += ` On every step: ${rng(full0)} (${full0.length} run${full0.length > 1 ? 's' : ''}).`; fullS.forEach((x) => { const b2 = rows.find((y) => y.a.kind === 'baseline2' && y.a.seed === x.a.seed); t += ` In a second seed${x.a.steps > base.a.steps ? `, ${x.a.steps} steps` : ''}: ${sg1(x.b.shift)}${b2 && b2.a.steps === x.a.steps ? `, where plain RMU gave back ${f1(b2.b.shift)}` : ''}.`; }); if (gone.length) { const m = gone.reduce((q, x) => (Math.abs(x.a.forget - base.a.forget) < Math.abs(q.a.forget - base.a.forget) ? x : q)); const other = D.formats.map((f) => f.id).filter((p) => p !== FP && p !== prec && full.every((x) => get(x.a.id, p)))[0]; if (other) { const v = full.map((x) => get(x.a.id, other).shift * 100), lo = Math.min.apply(null, v), hi = Math.max.apply(null, v); if (hi >= GREW * 100 || lo <= -GREW * 100) t += ` It repairs the format it trained on: in ${fmtLabel(other)}, which training never showed, the same runs moved ${sg1(lo / 100)} to ${sg1(hi / 100)} points.`; } t += ` It is not free: the one closest to the plain run at ${fpl} (${f1(m.a.forget)}% against ${f1(base.a.forget)}%) scores ${f1(m.b.forget)}% in the file${m.a.steps > base.a.steps ? `, after ${m.a.steps} steps instead of ${base.a.steps},` : ''} and lost ${f1(m.b.price)} points of general knowledge there. The plain run lost ${f1(base.b.price)}.`; } $('fix-take').textContent = t; const lo = 0.20, hi = Math.max(0.56, Math.ceil(Math.max.apply(null, rows.map((x) => Math.max(x.a.forget, x.b.forget))) * 20) / 20 + 0.02); const X = (v) => (clamp((v - lo) / (hi - lo), 0, 1) * 100).toFixed(2) + '%'; const box = $('fixes'); rows.forEach((x) => { const d = x.b.shift, tone = d >= RISE - 1e-9 ? 'hot' : d >= GREW ? 'mid' : 'ok'; const a = Math.min(x.a.forget, x.b.forget), w = Math.abs(x.b.forget - x.a.forget); const r = el('div', 'fx'); r.setAttribute('role', 'listitem'); r.innerHTML = `${esc(x.a.label)}${x.a.steps ? x.a.steps + ' steps' + (x.a.file_steps ? `, ${x.a.file_steps} as the file` : '') + ' \u00b7 ' : ''}${f1(x.b.price)} points of general knowledge lost in the file` + `` + `${sg1(d)}${f1(x.a.forget)}% at ${esc(fpl)}, ${f1(x.b.forget)}% in ${esc(fl)}`; r.setAttribute('aria-label', `${x.a.label}: ${f1(x.a.forget)} percent at ${fpl}, ${f1(x.b.forget)} percent in ${fl}, ${sg1(d)} points`); box.appendChild(r); }); $('fix-key').innerHTML = `${esc(D.benchmark.forget)} at ${esc(fpl)}in the ${esc(fl)} fileThe bar between them is what came back. The line marks guessing (${pct(CH)}%). All runs: layers ${rangeOf((D.families.deeper || {}).layers)}${rows.some((x) => x.a.seed) ? '' : ', same seed'}, scored as round trips of the decoder weights${MET.deeper_file ? `; the ${esc(methodShort('deeper_file'))} tab at the top shows one of them as real files` : ''}.`; } /* only when the run produced a robustness array: pushed harder, per method and attack */ const ATTACK = { zero_shot: 'Plain question', five_shot: 'Five worked examples', role_play: 'Expert persona', relearn_25_steps_64_questions: '25 fine-tuning steps on 64 hazard questions', relearn_25_steps_64_unrelated_questions: '25 fine-tuning steps on 64 unrelated questions' }; function buildRobustness() { const R = D.robustness; if (!Array.isArray(R) || !R.length) return; const att = Object.keys(ATTACK).filter((k) => R.some((r) => r.attack === k)).concat(Array.from(new Set(R.map((r) => r.attack))).filter((k) => !ATTACK[k])); const precs = Array.from(new Set(R.map((r) => r.prec))); const ms = [ORIG].concat(NOTORIG.map((m) => m.id)).filter((m) => R.some((r) => r.method === m)); const get = (m, p, k) => R.find((r) => r.method === m && r.prec === p && r.attack === k); $('rob-wrap').hidden = false; const aName = (k) => ATTACK[k] || k.replace(/_/g, ' '); const isRe = (k) => /relearn|fine/.test(k); const near = (m, p, k) => { const r = get(m, p, k), o = get(ORIG, p, k); return r && o ? { r, o, fr: (r.acc - CH) / Math.max(0.001, o.acc - CH), m, p, k } : null; }; const all = []; NOTORIG.forEach((m) => precs.forEach((p) => att.forEach((k) => { const x = near(m.id, p, k); if (x) all.push(x); }))); const pr = all.filter((x) => !isRe(x.k)), fi = all.filter((x) => isRe(x.k)); const promptsBack = pr.some((x) => x.fr >= 0.5), ftBack = fi.some((x) => x.fr >= 0.5); const unrel = (k) => /unrelated/.test(k), fiT = fi.filter((x) => !unrel(x.k)), fiU = fi.filter((x) => unrel(x.k)); const topicBack = fiT.some((x) => x.fr >= 0.5), unrelBack = fiU.some((x) => x.fr >= 0.5); $('rob-head').textContent = !fi.length ? 'Pushed harder: prompts and a persona' : (!promptsBack && topicBack && fiU.length && !unrelBack) ? 'Pushed harder: prompts did not bring it back. Fine-tuning on the topic did.' : (!promptsBack && ftBack) ? 'Pushed harder: prompts did not bring it back. Fine-tuning did.' : (promptsBack && ftBack) ? 'Pushed harder: prompts and fine-tuning both brought some of it back.' : promptsBack ? 'Pushed harder: prompts brought some of it back.' : 'Pushed harder: neither prompts nor fine-tuning brought it back.'; const rng = (xs) => { const v = xs.map((x) => x.r.acc), lo = Math.min.apply(null, v), hi = Math.max.apply(null, v); return `${pct(lo)}% to ${pct(hi)}%`; }; const names = (ks) => { const n = ks.map((k) => aName(k).toLowerCase()); return n.length > 1 ? n.slice(0, -1).join(', ') + ' and ' + n[n.length - 1] : n[0]; }; let cap = ''; if (pr.length) { const pk = Array.from(new Set(pr.map((x) => x.k))), po = pr.map((x) => ({ r: x.o })); cap += promptsBack ? `Under ${names(pk)}, some methods come back: ${pr.filter((x) => x.fr >= 0.5).map((x) => `${esc(methodShort(x.m))} reaches ${pct(x.r.acc)}% (original ${pct(x.o.acc)}%)`).join('; ')}. ` : `Under ${names(pk)}, every unlearned model stays at ${rng(pr)} on ${D.benchmark.forget} while the original scores ${rng(po)}. `; } Array.from(new Set(fi.map((x) => x.k))).forEach((fk) => { const g = fi.filter((x) => x.k === fk), back = g.some((x) => x.fr >= 0.5), fo = g[0].o.acc; cap += `${aName(fk)} ${back ? 'brings most of it back' : 'does not bring it back'}: ` + g.map((x) => `${esc(methodShort(x.m))} ${pct(x.r.acc)}%${back ? ` (${pct(clamp(x.fr, 0, 1))}% of the way back)` : ''}`).join(', ') + `, against ${pct(fo)}% for the original after the same fine-tuning${g.some((x) => x.p !== g[0].p) ? '' : ' at ' + esc(fmtLabel(g[0].p))}. `; }); cap += `The line on each bar marks guessing (${pct(CH)}%).`; $('rob-cap').innerHTML = cap; precs.sort((x, y) => D.formats.findIndex((f) => f.id === x) - D.formats.findIndex((f) => f.id === y)); let curP = precs.find((p) => R.some((r) => r.prec === p && isRe(r.attack))) || precs[0]; const tabs = $('rob-tabs'), box = $('rob'); const draw2 = () => { tabs.querySelectorAll('button').forEach((b) => b.setAttribute('aria-pressed', b.dataset.p === curP ? 'true' : 'false')); box.innerHTML = att.filter((k) => R.some((r) => r.prec === curP && r.attack === k)).map((k) => { const o = get(ORIG, curP, k); const rows = ms.map((m) => { const r = get(m, curP, k); if (!r) return ''; const w = (r.acc / 0.7 * 100).toFixed(1), fr = o && m !== ORIG ? (r.acc - CH) / Math.max(0.001, o.acc - CH) : 0, cls = m === ORIG ? 'o' : fr >= 0.5 ? 'hot' : 'ok'; return `
${esc(methodShort(m))}${pct(r.acc)}%
`; }).join(''); return `

${esc(aName(k))}

${rows}
`; }).join(''); }; precs.forEach((p) => { const b = el('button', 'mt'); b.type = 'button'; b.textContent = fmtLabel(p); b.dataset.p = p; b.addEventListener('click', () => { curP = p; draw2(); }); tabs.appendChild(b); }); draw2(); } /* every run we made: general knowledge kept against hazardous biology still answered */ const HEAD = { 'd_lr4e-5': 'Default layers', 'c_lr1.5e-4': 'CircuitKIT layers', 'c_lr1.5e-4_qa': 'CircuitKIT + 4-bit aware', 'd_lr1e-4': 'Aggressive' }; const THRESH = 0.40; function buildFrontier() { const P = D.frontier; if (!Array.isArray(P) || !P.length) return; const fams = ['default', 'circuit']; const cnt = (p, f) => P.filter((x) => x.prec === p && x.family === f).length; const precs = D.formats.map((f) => f.id).filter((p) => fams.every((f) => cnt(p, f) >= 5)); if (!precs.length) return; $('fr-wrap').hidden = false; let cur = precs.indexOf('NF4') >= 0 ? 'NF4' : precs[0], tableOn = false, active = null; const famName = { default: methodShort('standard') || 'Default layers', circuit: methodShort('circuit') || 'CircuitKIT layers', original: 'Original' }; const FAMS = D.families || {}; const famLabel = (f) => (f === 'original' ? 'Original' : (FAMS[f] && FAMS[f].label) || famName[f] || f); const ctl = (f) => famLabel(f).replace(/^Control:\s*/i, ''); const lowName = (f) => (f === 'default' ? famName.default.toLowerCase() : famName[f] || ctl(f).toLowerCase()); const known = ['default', 'circuit', 'shallower', 'deeper', 'deepest']; const order = known.filter((f) => P.some((p) => p.family === f)).concat(Array.from(new Set(P.map((p) => p.family))).filter((f) => f !== 'original' && known.indexOf(f) < 0)); const controls = order.filter((f) => f !== 'default' && f !== 'circuit'); const nameOf = (p) => HEAD[p.id] || (p.seed === 1 ? 'seed 2' : '') || ''; const xs = P.map((p) => p.forget).concat([CH]), ys = P.map((p) => p.retain); const x0 = Math.floor((Math.min.apply(null, xs) - 0.015) * 20) / 20, x1 = Math.ceil((Math.max.apply(null, xs) + 0.015) * 20) / 20; const y0 = Math.floor((Math.min.apply(null, ys) - 0.015) * 20) / 20, y1 = Math.ceil((Math.max.apply(null, ys) + 0.015) * 20) / 20; const tabs = $('fr-tabs'); precs.forEach((p) => { const b = el('button', 'mt'); b.type = 'button'; b.textContent = fmtLabel(p); b.dataset.p = p; b.addEventListener('click', () => { cur = p; draw(); }); tabs.appendChild(b); }); $('fr-view').addEventListener('click', () => { tableOn = !tableOn; draw(); }); const sentence = () => { const fl = fmtLabel(cur), all = (f) => P.filter((x) => x.prec === cur && x.family === f), pts = (f) => all(f).filter((x) => x.forget <= THRESH); const pf = (x) => (x < 0 ? '\u2212' : '') + Math.abs(x).toFixed(1); const span = (a) => { const v = a.map((x) => x.price * 100), lo = Math.min.apply(null, v), hi = Math.max.apply(null, v); return pf(lo) === pf(hi) ? `${pf(lo)} points` : `${pf(lo)} to ${pf(hi)} points`; }; const rangeTxt = (f) => { const a = pts(f); return a.length ? `${lowName(f)} cost ${span(a)} of general knowledge (${a.length} run${a.length > 1 ? 's' : ''})` : `no ${lowName(f)} run got there`; }; const d = pts('default'), c = pts('circuit'); let t = `Among runs that got ${D.benchmark.forget} to ${pct(THRESH)}% or below at ${fl}, ${rangeTxt('default')}; ${rangeTxt('circuit')}.`; const others = [['default', d]], missed = []; controls.forEach((f) => { const al = all(f); if (!al.length) return; const a = pts(f); if (a.length) { others.push([f, a]); t += ` ${ctl(f).charAt(0).toUpperCase() + ctl(f).slice(1)}: ${span(a)} (${a.length} run${a.length > 1 ? 's' : ''}).`; } else { const best = al.reduce((m, x) => (x.forget < m.forget ? x : m)); missed.push(f); t += ` No run at ${ctl(f).toLowerCase()} got to ${pct(THRESH)}% or below; its closest run reached ${pct(best.forget)}% for ${pf(best.price * 100)} points.`; } }); let cheaper = false; if (c.length && d.length) { const cMax = Math.max.apply(null, c.map((x) => x.price)); cheaper = others.every(([, a]) => a.length && cMax < Math.min.apply(null, a.map((x) => x.price))); if (cheaper) t += ` The dearest CircuitKIT run costs less than the cheapest run of every other family that got there.`; } return { t, cheaper }; }; function draw() { tabs.querySelectorAll('.mt').forEach((b) => b.setAttribute('aria-pressed', b.dataset.p === cur ? 'true' : 'false')); $('fr-view').setAttribute('aria-pressed', tableOn ? 'true' : 'false'); $('fr-view').textContent = tableOn ? 'View as chart' : 'View as table'; const s = sentence(); $('fr-head').textContent = s.cheaper ? 'Every run we made: the CircuitKIT layers keep more at the same level of forgetting' : 'Every run we made, by where the edit was placed'; $('fr-take').textContent = s.t; const pts = P.filter((p) => p.prec === cur); $('fr-chart').hidden = tableOn; $('fr-table').hidden = !tableOn; if (tableOn) { const rows = pts.slice().sort((a, b) => (order.indexOf(a.family) + 1 || 9) - (order.indexOf(b.family) + 1 || 9) || a.forget - b.forget); $('fr-table').innerHTML = `${rows.map((p) => ``).join('')}
RunFamilySeed4-bit aware${esc(D.benchmark.forget)}${esc(D.benchmark.retain)}Price, points
${esc(HEAD[p.id] || p.id)}${esc(famLabel(p.family))}${p.seed + 1}${p.qa ? 'yes' : 'no'}${p1(p.forget)}%${p1(p.retain)}%${(p.price * 100).toFixed(1)}
`; return; } render(pts); drawKey(); } function render(pts) { const box = $('fr-chart'), W = clamp(Math.round(box.clientWidth - 16 || 640), 300, 1100), H = W < 520 ? 340 : 430; const L = W < 520 ? 44 : 56, R = 14, T = 22, B = 46, iw = W - L - R, ih = H - T - B; const X = (v) => L + (v - x0) / (x1 - x0) * iw, Y = (v) => T + (1 - (v - y0) / (y1 - y0)) * ih; const orig = pts.find((p) => p.family === 'original'); let g = ``; for (let v = Math.ceil(x0 * 20) / 20; v <= x1 + 1e-9; v += 0.05) g += `${pct(v)}`; for (let v = Math.ceil(y0 * 20) / 20; v <= y1 + 1e-9; v += 0.05) g += `${pct(v)}`; g += `${esc(D.benchmark.forget)} still answered, % (lower is more forgotten)`; g += `${esc(D.benchmark.retain)} kept, %`; g += `guessing ${pct(CH)}%`; if (orig) g += `original keeps ${pct(orig.retain)}%`; // marks const lab = []; pts.forEach((p, i) => { const cx = X(p.forget), cy = Y(p.retain), isCtl = controls.indexOf(p.family) >= 0, col = p.family === 'default' ? 'var(--hot)' : p.family === 'circuit' ? 'var(--ok)' : isCtl ? 'var(--paper-ink-muted)' : 'var(--bar)'; const hollow = p.seed === 1, fill = hollow ? 'var(--paper)' : isCtl ? 'oklch(88% .01 70)' : col, sw = hollow ? 2.5 : isCtl ? 2 : 2, st2 = hollow || isCtl ? col : 'var(--paper)'; const label = `${HEAD[p.id] || (p.family === 'original' ? 'Original' : p.id)}${p.seed === 1 ? ', seed 2' : ''}${p.qa ? ', 4-bit aware' : ''}: ${esc(D.benchmark.forget)} ${p1(p.forget)}%, ${esc(D.benchmark.retain)} ${p1(p.retain)}%, price ${(p.price * 100).toFixed(1)} points`; let shape; if (p.family === 'circuit') shape = ``; else if (p.family === 'original') shape = ``; else if (p.family === 'deeper') shape = ``; else if (p.family === 'shallower') shape = ``; else if (p.family === 'deepest') shape = ``; else shape = ``; if (p.qa) shape += ``; g += `${shape}`; const nm = p.family === 'original' ? 'Original' : (HEAD[p.id] || (p.seed === 1 ? 'seed 2' : '')); if (nm && !(W < 520 && (p.qa || nm === 'seed 2'))) lab.push({ x: cx, y: cy, t: nm, qa: p.qa, s1: p.seed === 1 && !HEAD[p.id] }); }); // direct labels with simple collision nudging const placed = []; lab.sort((a, b) => a.y - b.y).forEach((l) => { const w = l.t.length * 6.4, right = l.x + 10 + w > W - R; let x = right ? l.x - 10 : l.x + 10, y = l.qa ? l.y + 18 : l.y - 9, anc = right ? 'end' : 'start'; for (let k = 0; k < 8 && placed.some((q) => Math.abs(q.y - y) < 13 && ((x < q.x + q.w && x + w > q.x) || (anc === 'end' && x - w < q.x + q.w && x > q.x))); k++) y += 13; placed.push({ x: anc === 'end' ? x - w : x, y, w }); g += `${esc(l.t)}`; }); g += `better: up and to the left`; box.innerHTML = g; const tip = $('fr-tip'), svg = box.querySelector('svg'); const show = (i) => { const p = pts[i]; if (!p) return; box.querySelectorAll('.pt').forEach((n) => n.classList.toggle('on', +n.dataset.i === i)); const nm = HEAD[p.id] || (p.family === 'original' ? 'Original' : p.id); tip.innerHTML = `${esc(nm)}${p.family === 'original' ? 'no unlearning' : esc(famLabel(p.family)) + (p.seed === 1 ? ', seed 2' : '') + (p.qa ? ', 4-bit aware' : '')}${esc(D.benchmark.forget)} ${p1(p.forget)}%${esc(D.benchmark.retain)} ${p1(p.retain)}%Price ${(p.price * 100).toFixed(1)} points`; tip.hidden = false; const cx = X(p.forget), cy = Y(p.retain), flip = cx > W * 0.6; tip.style.left = (cx / W * 100) + '%'; tip.style.top = (cy / H * 100) + '%'; tip.style.transform = `translate(${flip ? 'calc(-100% - 14px)' : '14px'}, ${cy > H * 0.6 ? 'calc(-100% - 8px)' : '8px'})`; }; const hide = () => { tip.hidden = true; box.querySelectorAll('.pt').forEach((n) => n.classList.remove('on')); }; svg.addEventListener('pointermove', (e) => { const r = svg.getBoundingClientRect(), k = W / r.width, mx = (e.clientX - r.left) * k, my = (e.clientY - r.top) * k; let best = -1, bd = 24 * 24; pts.forEach((p, i) => { const dx = X(p.forget) - mx, dy = Y(p.retain) - my, dd = dx * dx + dy * dy; if (dd < bd) { bd = dd; best = i; } }); if (best >= 0) show(best); else hide(); }); svg.addEventListener('pointerleave', hide); box.querySelectorAll('.pt').forEach((n) => { n.addEventListener('focus', () => show(+n.dataset.i)); n.addEventListener('blur', hide); n.addEventListener('keydown', (e) => { if (e.key === 'Escape') { hide(); n.blur(); } }); }); } // legend: shape and fill carry identity, not colour alone; only families present at the selected format const ic = (svg) => ``; const ICON = { default: '', circuit: '', original: '', deeper: '', shallower: '', deepest: '' }; const drawKey = () => { const have = (f) => P.some((p) => p.prec === cur && p.family === f); const items = order.filter(have).map((f) => ic(ICON[f] || '') + esc(famLabel(f))); items.push(ic(ICON.original) + 'Original'); if (P.some((p) => p.prec === cur && p.seed === 1)) items.push(ic('') + 'Hollow: second seed'); if (P.some((p) => p.prec === cur && p.qa)) items.push(ic('') + 'Dot: trained through a 4-bit quantizer'); $('fr-key').innerHTML = items.map((x) => `
  • ${x}
  • `).join(''); }; let rt; window.addEventListener('resize', () => { clearTimeout(rt); rt = setTimeout(() => { if (!tableOn) draw(); }, 150); }); draw(); } function markTable() { document.querySelectorAll('#grid tbody tr').forEach((r) => r.classList.toggle('sel', r.dataset.f === st.f)); } function wireStory() { const pipe = $('pipe'), steps = Array.from(pipe.querySelectorAll('.step')); if (reduced() || !('IntersectionObserver' in window)) { pipe.classList.add('static'); return; } pipe.classList.add('go'); const io = new IntersectionObserver((es) => { if (es.some((e) => e.isIntersecting)) { steps.forEach((s, i) => setTimeout(() => s.classList.add('lit'), i * 320)); io.disconnect(); } }, { threshold: 0.2 }); io.observe(pipe); } })();