/* HERB viewer β€” corpus, complete eval set, six answerable-only runs, and comparison. */ (function () { "use strict"; var $ = function (id) { return document.getElementById(id); }; var HERB_BASE = "https://huggingface.co/datasets/Salesforce/HERB/resolve/main/products/"; var RECORD_CACHE_LIMIT = 8; var ARTIFACT_TYPES = [ { key: "slack", label: "Slack", icon: "πŸ’¬" }, { key: "documents", label: "Documents", icon: "πŸ“„" }, { key: "meeting_transcripts", label: "Meeting transcripts", icon: "πŸŽ™" }, { key: "meeting_chats", label: "Meeting chats", icon: "πŸ’­" }, { key: "urls", label: "URLs", icon: "πŸ”—" }, { key: "prs", label: "Pull requests", icon: "πŸ”€" }, ]; var TYPE_LABEL = {}; ARTIFACT_TYPES.forEach(function (t) { TYPE_LABEL[t.key] = t; }); function esc(s) { if (s === null || s === undefined) return ""; return String(s).replace(/&/g, "&").replace(//g, ">").replace(/"/g, """); } function truncate(s, n) { s = String(s || "").replace(/\s+/g, " ").trim(); return s.length > n ? s.slice(0, n) + "…" : s; } var state = { mode: "corpus", products: [], productByName: {}, employees: {}, customers: {}, curProduct: null, productCache: {}, curType: null, artifactsView: [], artifactIdx: 0, pendingArtifactId: null, loadSeq: 0, questions: [], evalView: [], evalIdx: 0, manifest: null, runBySlot: {}, runIndexes: {}, runIndexPromises: {}, runRecords: {}, runRecordOrder: [], runRecordPromises: {}, runSlot: null, runView: [], runIdx: 0, runLoadSeq: 0, runFilterSeq: 0, compareIndex: null, compareIndexPromise: null, compareRecords: {}, compareRecordOrder: [], compareRecordPromises: {}, compareView: [], compareIdx: 0, compareLoadSeq: 0, compareFilterSeq: 0, }; function cachedRecord(cache, order, key) { if (!Object.prototype.hasOwnProperty.call(cache, key)) return null; var index = order.indexOf(key); if (index >= 0) order.splice(index, 1); order.push(key); return cache[key]; } function storeRecord(cache, order, key, record) { cache[key] = record; var index = order.indexOf(key); if (index >= 0) order.splice(index, 1); order.push(key); while (order.length > RECORD_CACHE_LIMIT) delete cache[order.shift()]; } /* ---------------- id resolution ---------------- */ function resolveEid(eid) { var e = state.employees[eid]; return e ? e.name : eid; } function eidTitle(eid) { var e = state.employees[eid]; return e ? (e.role || "") + (e.org ? " Β· " + e.org : "") : ""; } // escape text, then turn eid_xxxx tokens into resolved name tags function escResolve(text) { var out = esc(text); out = out.replace(/eid_[0-9a-f]{6,}/g, function (m) { return '' + esc(resolveEid(m)) + ""; }); return out; } function normalizeQid(value) { return String(value || "").replace(/#a(\d+)$/, "_a$1"); } function rememberQid(value) { var qid = normalizeQid(value); if (qid) window.TrajectoryUI.setQid(qid); } function questionMeta(value) { return window.HerbQuestionMetaByQid[normalizeQid(value)] || {}; } function computationFilterMatches(value, filter) { if (!filter) return true; return Boolean(questionMeta(value).computation_comparison) === (filter === "yes"); } function computationBadge(record) { var meta = record.computation_comparison === undefined ? questionMeta(record.qid || record.gid) : record; var isComputation = Boolean(meta.computation_comparison); var label = isComputation ? "computation / comparison" : "other"; return '' + label + ""; } function sharedIndex(records) { var qid = window.TrajectoryUI.getQid(); if (!qid) return -1; return records.findIndex(function (record) { return normalizeQid(record.qid || record.gid) === qid; }); } function displayGold(value) { var values = Array.isArray(value) ? value : [value]; return values.map(function (item) { return typeof item === "string" && /^eid_[0-9a-f]+$/.test(item) ? resolveEid(item) : item; }); } window.HerbDisplayGold = displayGold; /* ---------------- data loading ---------------- */ function loadAll() { return Promise.all([ fetch("products.json").then(function (r) { return r.json(); }), fetch("eval.json").then(function (r) { return r.json(); }), fetch("employees.json").then(function (r) { return r.json(); }), fetch("customers.json").then(function (r) { return r.json(); }), fetch("runs/manifest.json").then(function (r) { if (!r.ok) throw new Error("run manifest HTTP " + r.status); return r.json(); }), ]).then(function (res) { state.products = res[0] || []; state.questions = res[1] || []; state.employees = res[2] || {}; (res[3] || []).forEach(function (c) { state.customers[c.id] = c; }); state.manifest = res[4] || { runs: [] }; (state.manifest.runs || []).forEach(function (run) { state.runBySlot[run.slot] = run; }); state.products.forEach(function (p) { state.productByName[p.name] = p; }); window.HerbQuestionMetaByQid = {}; state.questions.forEach(function (question) { window.HerbQuestionMetaByQid[normalizeQid(question.gid)] = question; }); }); } function loadProduct(name) { if (state.productCache[name]) return Promise.resolve(state.productCache[name]); return fetch(HERB_BASE + name + ".json").then(function (r) { if (!r.ok) throw new Error("HTTP " + r.status); return r.json(); }).then(function (d) { var byType = {}, byId = {}; ARTIFACT_TYPES.forEach(function (t) { var items = d[t.key] || []; byType[t.key] = items; items.forEach(function (it) { if (it && it.id != null && !(it.id in byId)) byId[it.id] = { type: t.key, item: it }; }); }); var rec = { data: d, byType: byType, byId: byId }; state.productCache[name] = rec; return rec; }); } /* ---------------- corpus tab ---------------- */ function populateProductSelects() { var cp = $("corpusProduct"), ep = $("evalProduct"), rp = $("runProduct"), xp = $("compareProduct"); state.products.forEach(function (p) { var o = document.createElement("option"); o.value = p.name; o.textContent = p.name + " (" + p.n_artifacts + ")"; cp.appendChild(o); var o2 = document.createElement("option"); o2.value = p.name; o2.textContent = p.name; ep.appendChild(o2); var o3 = document.createElement("option"); o3.value = p.name; o3.textContent = p.name; rp.appendChild(o3); var o4 = document.createElement("option"); o4.value = p.name; o4.textContent = p.name; xp.appendChild(o4); }); ["person", "content", "company", "pr", "url"].forEach(function (typ) { var r = document.createElement("option"); r.value = typ; r.textContent = typ; $("runType").appendChild(r); var x = document.createElement("option"); x.value = typ; x.textContent = typ; $("compareType").appendChild(x); }); } function populateTypeSelect(rec) { var sel = $("artifactType"); sel.innerHTML = ""; ARTIFACT_TYPES.forEach(function (t) { var n = (rec.byType[t.key] || []).length; if (!n) return; var o = document.createElement("option"); o.value = t.key; o.textContent = t.icon + " " + t.label + " (" + n + ")"; sel.appendChild(o); }); } function onProductChange() { var name = $("corpusProduct").value; state.curProduct = name; var seq = ++state.loadSeq; $("artifactMeta").innerHTML = ""; $("artifactBody").className = "artifact-body loading"; $("artifactBody").textContent = "Loading " + name + "…"; loadProduct(name).then(function (rec) { if (seq !== state.loadSeq) return; populateTypeSelect(rec); // choose type: pending artifact's type, else current if still valid, else first var chosen = null; if (state.pendingArtifactId != null && rec.byId[state.pendingArtifactId]) { chosen = rec.byId[state.pendingArtifactId].type; } else if (state.curType && (rec.byType[state.curType] || []).length) { chosen = state.curType; } if (!chosen) chosen = $("artifactType").options.length ? $("artifactType").options[0].value : null; state.curType = chosen; $("artifactType").value = chosen || ""; $("corpusFilter").value = ""; filterArtifacts(); }).catch(function (err) { if (seq !== state.loadSeq) return; $("artifactBody").className = "artifact-body"; $("artifactBody").innerHTML = '
Failed to load product: ' + esc(err.message) + "
"; }); } function artifactTitle(type, it) { switch (type) { case "slack": var ch = (it.Channel && it.Channel.name) ? "#" + it.Channel.name + ": " : ""; var txt = (it.Message && it.Message.User && it.Message.User.text) || ""; return ch + truncate(txt, 60) || it.id; case "documents": return (it.type ? it.type + " β€” " : "") + it.id; case "meeting_transcripts": return (it.document_type ? it.document_type + " β€” " : "") + it.id; case "meeting_chats": return it.id; case "urls": return it.description ? truncate(it.description, 70) : (it.link || it.id); case "prs": return (it.number ? "#" + it.number + " " : "") + (it.title || it.id); default: return it.id; } } function filterArtifacts() { var rec = state.productCache[state.curProduct]; if (!rec || !state.curType) { state.artifactsView = []; } else { var q = $("corpusFilter").value.trim().toLowerCase(); var items = rec.byType[state.curType] || []; state.artifactsView = items.map(function (it) { return { item: it, title: artifactTitle(state.curType, it) }; }) .filter(function (a) { return !q || a.title.toLowerCase().indexOf(q) !== -1 || String(a.item.id).toLowerCase().indexOf(q) !== -1; }); } if (state.artifactIdx >= state.artifactsView.length) state.artifactIdx = 0; rebuildArtifactSelect(); renderArtifact(); } function rebuildArtifactSelect() { var sel = $("artifactSelect"); sel.innerHTML = ""; state.artifactsView.forEach(function (a, i) { var o = document.createElement("option"); o.value = String(i); o.textContent = a.title; sel.appendChild(o); }); sel.value = String(state.artifactIdx); } function kvRow(pairs) { var rows = pairs.filter(function (p) { return p[1] !== undefined && p[1] !== null && p[1] !== ""; }) .map(function (p) { return '
' + esc(p[0]) + '
' + p[1] + "
"; }).join(""); return rows ? '
' + rows + "
" : ""; } function renderSlackReply(rep) { var u = (rep.Message && rep.Message.User) || rep.User || rep; var who = u.userId ? resolveEid(u.userId) : (rep.sender ? resolveEid(rep.sender) : ""); var txt = u.text || rep.text || rep.message || ""; return '
' + esc(who) + "" + (u.timestamp ? '' + esc(u.timestamp) + "" : "") + '
' + escResolve(txt) + "
"; } function renderArtifactBody(type, it) { if (type === "slack") { var u = (it.Message && it.Message.User) || {}; var who = u.userId ? resolveEid(u.userId) : ""; var reacts = (it.Message && it.Message.Reactions) || []; var html = '
' + esc(who) + "" + (u.timestamp ? '' + esc(u.timestamp) + "" : "") + '
' + escResolve(u.text) + "
"; if (reacts.length) html += '
Reactions
' + esc(JSON.stringify(reacts)) + "
"; var replies = it.ThreadReplies || []; if (replies.length) { html += '
Thread replies (' + replies.length + ")
"; html += replies.map(renderSlackReply).join(""); } return html; } if (type === "documents") { var kv = kvRow([["Type", esc(it.type)], ["Author", escResolve(it.author)], ["Date", esc(it.date)]]); var body = window.marked ? window.marked.parse(it.content || "") : esc(it.content); return kv + '
Content
' + body + "
"; } if (type === "meeting_transcripts") { var parts = (it.participants || []).map(function (e) { return '' + esc(resolveEid(e)) + ""; }).join(", "); var kv2 = kvRow([["Meeting type", esc(it.document_type)], ["Date", esc(it.date)], ["Participants", parts]]); return kv2 + '
Transcript
' + escResolve(it.transcript) + "
"; } if (type === "meeting_chats") { return '
Chat
' + escResolve(it.text) + "
"; } if (type === "urls") { return kvRow([["Link", '' + esc(it.link) + " β†—"], ["Description", escResolve(it.description)]]); } if (type === "prs") { var kv3 = kvRow([ ["Number", esc(it.number)], ["State", '' + esc(it.state) + ""], ["Merged", esc(it.merged)], ["Mergeable", esc(it.mergeable)], ["Author", esc(it.user && it.user.login)], ["Created", esc(it.created_at)], ["Link", it.link ? '' + esc(it.link) + " β†—" : ""], ]); var summary = it.summary ? '
Summary
' + escResolve(it.summary) + "
" : ""; var reviews = ""; if (it.reviews) { var parsed = null; try { parsed = JSON.parse(String(it.reviews).replace(/'/g, '"')); } catch (e) { parsed = null; } if (parsed && parsed.length) { reviews = '
Reviews (' + parsed.length + ")
" + parsed.map(function (r) { return '
' + esc(r.state) + " " + esc(r.user && r.user.login) + (r.submitted_at ? ' Β· ' + esc(r.submitted_at) + "" : "") + (r.comment ? '
' + escResolve(r.comment) + "
" : "") + "
"; }).join(""); } else { reviews = '
Reviews
' + esc(it.reviews) + "
"; } } return kv3 + summary + reviews; } return '
' + esc(JSON.stringify(it, null, 2)) + "
"; } function renderArtifact() { var meta = $("artifactMeta"), body = $("artifactBody"); if (!state.artifactsView.length) { meta.innerHTML = '
No artifacts match.
'; body.className = "artifact-body"; body.innerHTML = ""; $("corpusCounter").textContent = "0 / 0"; return; } var a = state.artifactsView[state.artifactIdx]; var it = a.item, type = state.curType; $("artifactSelect").value = String(state.artifactIdx); $("corpusCounter").textContent = (state.artifactIdx + 1) + " / " + state.artifactsView.length; var tl = TYPE_LABEL[type]; var link = ""; if (it.link) link = 'open link β†—'; else if (it.document_link) link = 'source β†—'; meta.innerHTML = "

" + esc(a.title) + "

" + '
' + 'Product: ' + esc(state.curProduct) + "" + '' + esc(tl.label) + "" + '' + esc(it.id) + "" + link + "
"; body.className = "artifact-body"; body.innerHTML = renderArtifactBody(type, it); body.scrollTop = 0; } function selectArtifact(product, artifactId) { $("corpusProduct").value = product; state.pendingArtifactId = artifactId; state.artifactIdx = 0; // onProductChange will honor pendingArtifactId to pick the type; then locate the item var seq = ++state.loadSeq; state.curProduct = product; $("artifactBody").className = "artifact-body loading"; $("artifactBody").textContent = "Loading " + product + "…"; loadProduct(product).then(function (rec) { if (seq !== state.loadSeq) return; populateTypeSelect(rec); var hit = rec.byId[artifactId]; state.curType = hit ? hit.type : ($("artifactType").options.length ? $("artifactType").options[0].value : null); $("artifactType").value = state.curType || ""; $("corpusFilter").value = ""; state.pendingArtifactId = null; // build view then locate index of the artifact var items = rec.byType[state.curType] || []; state.artifactsView = items.map(function (x) { return { item: x, title: artifactTitle(state.curType, x) }; }); var idx = 0; for (var i = 0; i < state.artifactsView.length; i++) { if (state.artifactsView[i].item.id === artifactId) { idx = i; break; } } state.artifactIdx = idx; rebuildArtifactSelect(); renderArtifact(); }); } /* ---------------- eval tab ---------------- */ function filterEval() { var q = $("evalSearch").value.trim().toLowerCase(); var prod = $("evalProduct").value; var kind = $("kindFilter").value; var typ = $("qTypeFilter").value; var computationFilter = $("evalComputationFilter").value; state.evalView = state.questions.filter(function (x) { if (prod && x.product !== prod) return false; if (kind && x.kind !== kind) return false; if (typ && x.type !== typ) return false; if (!computationFilterMatches(x.gid, computationFilter)) return false; if (q) { var hay = [x.question, x.gid, x.product, Array.isArray(x.ground_truth) ? x.ground_truth.join(" ") : x.ground_truth].join(" ").toLowerCase(); if (hay.indexOf(q) === -1) return false; } return true; }); var target = sharedIndex(state.evalView); if (target >= 0) state.evalIdx = target; else if (state.evalIdx >= state.evalView.length) state.evalIdx = 0; rebuildQSelect(); renderEval(); } function rebuildQSelect() { var sel = $("qSelect"); sel.innerHTML = ""; state.evalView.forEach(function (x, i) { var o = document.createElement("option"); o.value = String(i); o.textContent = "[" + x.product + (x.kind === "unanswerable" ? " ⊘" : "") + "] " + truncate(x.question, 58); sel.appendChild(o); }); sel.value = String(state.evalIdx); } function field(lbl, valHtml) { return '
' + esc(lbl) + '
' + valHtml + "
"; } function renderGroundTruth(gt) { if (Array.isArray(gt)) { return '"; } return escResolve(gt); } function renderEval() { var card = $("evalCard"); if (!state.evalView.length) { card.innerHTML = '
No questions match.
'; $("evalCounter").textContent = "0 / 0"; return; } var x = state.evalView[state.evalIdx]; rememberQid(x.gid); $("qSelect").value = String(state.evalIdx); $("evalCounter").textContent = (state.evalIdx + 1) + " / " + state.evalView.length; var badges = '' + esc(x.gid) + "" + '' + esc(x.product) + " β†—" + '' + esc(x.kind) + "" + (x.type && x.type !== "unanswerable" ? '' + esc(x.type) + "" : "") + computationBadge(x); var answerHtml; if (x.kind === "unanswerable") { answerHtml = '
Unanswerable
' + '
This question has no answer in the corpus (intentional distractor).
'; } else { answerHtml = window.TrajectoryUI.gold(displayGold(x.ground_truth), "Gold"); } var cites = x.citations || []; var citesHtml = cites.length ? '
' + cites.map(function (c) { return 'πŸ“Ž' + esc(c) + ""; }).join("") + "
" : 'none'; card.innerHTML = '
' + badges + "
" + "

" + esc(x.question) + "

" + answerHtml + (x.kind === "unanswerable" ? "" : field("Evidence citations (" + cites.length + ")", citesHtml)); Array.prototype.forEach.call(card.querySelectorAll(".cite-chip[data-cid]"), function (el) { el.addEventListener("click", function () { setMode("corpus"); selectArtifact(this.getAttribute("data-product"), this.getAttribute("data-cid")); }); }); var pb = card.querySelector(".badge.product[data-product]"); if (pb) pb.addEventListener("click", function () { setMode("corpus"); $("corpusProduct").value = this.getAttribute("data-product"); state.curType = null; onProductChange(); }); } /* ---------------- full-run + compare tabs ---------------- */ function buildTabButtons() { var tabs = [ { mode: "corpus", label: "πŸ“š Corpus" }, { mode: "eval", label: "❓ Eval" }, ]; (state.manifest.runs || []).forEach(function (run) { tabs.push({ mode: run.slot, label: run.label }); }); tabs.push({ mode: "compare", label: "βš– Compare" }); $("viewToggle").innerHTML = tabs.map(function (tab) { return '"; }).join(""); } function metricCard(label, value, detail, cls) { return '
' + esc(label) + '
' + esc(value) + '
' + esc(detail) + "
"; } function runStatus(record) { if (!record.answered) return { key: "missing", label: "missing / unanswered" }; if (record.failure && /judge/.test(record.failure)) { return { key: "failure", label: "judge failure" }; } if (record.correct) return { key: "correct", label: "perfect" }; if (record.score > 0) return { key: "partial", label: "partial credit" }; return { key: "incorrect", label: "incorrect" }; } function citationChips(product, citations) { if (!citations || !citations.length) return 'none'; return '
' + citations.map(function (cid) { return 'πŸ“Ž' + esc(cid) + ""; }).join("") + "
"; } function wireCitationClicks(container) { Array.prototype.forEach.call(container.querySelectorAll(".cite-chip[data-cid]"), function (el) { el.addEventListener("click", function () { setMode("corpus"); selectArtifact(this.getAttribute("data-product"), this.getAttribute("data-cid")); }); }); Array.prototype.forEach.call(container.querySelectorAll(".badge.product[data-product]"), function (el) { el.addEventListener("click", function () { setMode("corpus"); $("corpusProduct").value = this.getAttribute("data-product"); state.curType = null; onProductChange(); }); }); } function questionBadges(record, status) { return '
' + esc(record.gid || record.qid) + "" + '' + esc(record.product) + " β†—" + '' + esc(record.type) + "" + computationBadge(record) + (status ? '' + esc(status.label) + "" : "") + "
"; } function answerPanel(label, value, cls) { var html = value === null || value === undefined || value === "" ? 'none' : (Array.isArray(value) ? renderGroundTruth(value) : escResolve(value)); return '
' + esc(label) + '
' + html + "
"; } function runSummaryHtml(run) { return '
Scope: 815 answerable HERB questions only. ' + "Each run uses its canonical evaluator; missing or unanswered questions count as zero.
" + '
' + metricCard(run.score_label || "Score", run.score_pct.toFixed(2) + "%", run.score_detail || (run.correct + " / " + run.scope), "status-correct") + metricCard("Coverage", run.coverage_pct.toFixed(2) + "%", run.answered + " / " + run.scope + " answered", "") + metricCard("Missing", String(run.scope - run.answered), "explicit unanswered records", run.scope === run.answered ? "" : "status-missing") + "
"; } function ensureRunIndex(slot) { if (state.runIndexes[slot]) return Promise.resolve(state.runIndexes[slot]); if (state.runIndexPromises[slot]) return state.runIndexPromises[slot]; var request = fetch("runs/" + encodeURIComponent(slot) + "/index.json").then(function (r) { if (!r.ok) throw new Error("run index HTTP " + r.status); return r.json(); }).then(function (index) { state.runIndexes[slot] = index; delete state.runIndexPromises[slot]; return index; }, function (error) { delete state.runIndexPromises[slot]; throw error; }); state.runIndexPromises[slot] = request; return request; } function ensureRunRecord(slot, qid) { var key = slot + "/" + qid; var cached = cachedRecord(state.runRecords, state.runRecordOrder, key); if (cached) return Promise.resolve(cached); if (state.runRecordPromises[key]) return state.runRecordPromises[key]; var request = fetch("runs/" + encodeURIComponent(slot) + "/records/" + encodeURIComponent(qid) + ".json").then(function (r) { if (!r.ok) throw new Error("run record HTTP " + r.status); return r.json(); }).then(function (record) { delete state.runRecordPromises[key]; storeRecord(state.runRecords, state.runRecordOrder, key, record); return record; }, function (error) { delete state.runRecordPromises[key]; throw error; }); state.runRecordPromises[key] = request; return request; } function rebuildRunSelect() { var sel = $("runQSelect"); sel.innerHTML = ""; state.runView.forEach(function (item, i) { var option = document.createElement("option"); option.value = String(i); option.textContent = "[" + item.product + "] " + truncate(item.question, 58); sel.appendChild(option); }); sel.value = String(state.runIdx); } function filterRun() { var filterSeq = ++state.runFilterSeq; state.runLoadSeq++; var slot = state.runSlot; if (!slot) return; state.runView = []; $("runSummary").innerHTML = state.runBySlot[slot] ? runSummaryHtml(state.runBySlot[slot]) : ""; $("runQSelect").innerHTML = ""; $("runCounter").textContent = "0 / 0"; $("runCard").innerHTML = '
Loading run index…
'; ensureRunIndex(slot).then(function (index) { if (filterSeq !== state.runFilterSeq || state.mode !== slot) return; var q = $("runSearch").value.trim().toLowerCase(); var product = $("runProduct").value; var typ = $("runType").value; var status = $("runStatus").value; var computationFilter = $("runComputationFilter").value; state.runView = index.items.filter(function (item) { if (product && item.product !== product) return false; if (typ && item.type !== typ) return false; if (status === "correct" && !item.correct) return false; if (status === "partial" && !(item.answered && item.score > 0 && item.score < 1)) return false; if (status === "incorrect" && (!item.answered || item.correct || item.score > 0 || (item.failure && /judge/.test(item.failure)))) return false; if (status === "failure" && !(item.answered && item.failure && /judge/.test(item.failure))) return false; if (status === "missing" && item.answered) return false; if (!computationFilterMatches(item.qid, computationFilter)) return false; if (q) { var hay = [item.qid, item.gid, item.product, item.question, item.prediction, item.extracted_answer].join(" ").toLowerCase(); if (hay.indexOf(q) === -1) return false; } return true; }); var target = sharedIndex(state.runView); if (target >= 0) state.runIdx = target; else if (state.runIdx >= state.runView.length) state.runIdx = 0; rebuildRunSelect(); renderRun(); }).catch(function (err) { if (filterSeq === state.runFilterSeq && state.mode === slot) { $("runCard").innerHTML = '
Failed to load run: ' + esc(err.message) + "
"; } }); } function renderRun() { var seq = ++state.runLoadSeq; var run = state.runBySlot[state.runSlot]; $("runSummary").innerHTML = run ? runSummaryHtml(run) : ""; if (!state.runView.length) { $("runCard").innerHTML = '
No run questions match.
'; $("runCounter").textContent = "0 / 0"; return; } var item = state.runView[state.runIdx]; $("runQSelect").value = String(state.runIdx); $("runCounter").textContent = (state.runIdx + 1) + " / " + state.runView.length; $("runCard").innerHTML = '
Loading ' + esc(item.qid) + "…
"; var slot = state.runSlot; ensureRunRecord(slot, item.qid).then(function (record) { if (seq !== state.runLoadSeq || slot !== state.runSlot || state.mode !== slot) return; var status = runStatus(record); rememberQid(record.qid); var stop = record.stop_reason || record.finish_reason || (record.finish_reasons || []).slice(-1)[0] || "β€”"; var events = record.events || []; var eventHtml = window.TrajectoryUI.render(events, "Agent trajectory"); var citations = citationChips(record.product, record.citations); $("runCard").innerHTML = questionBadges(record, status) + "

" + esc(record.question) + "

" + window.TrajectoryUI.gold(displayGold(record.gold), "Gold") + (/^e2e_/.test(slot) ? window.TrajectoryUI.supporting( "Evidence citations", citations || 'none', (record.citations || []).length ) : field("Evidence citations (" + (record.citations || []).length + ")", citations)) + answerPanel("Prediction", record.prediction, "prediction") + answerPanel("Extracted judge answer", record.extracted_answer, "") + '
' + 'stop: ' + esc(stop) + "" + 'failure: ' + esc(record.failure || "none") + "" + 'confidence: ' + esc(record.confidence == null ? "β€”" : record.confidence) + "" + (typeof record.score === "number" ? 'canonical score: ' + (record.score * 100).toFixed(2) + "%" : "") + 'turns: ' + esc(record.turns == null ? "β€”" : record.turns) + "" + 'tokens: ' + esc((record.tokens.total_tokens || 0).toLocaleString()) + "" + 'tools: ' + esc(JSON.stringify(record.tool_counts || {})) + "" + "
" + answerPanel("Judge text", record.judge_text, "") + eventHtml; wireCitationClicks($("runCard")); window.TrajectoryUI.bind($("runCard")); }).catch(function (err) { if (seq === state.runLoadSeq && slot === state.runSlot && state.mode === slot) { $("runCard").innerHTML = '
Failed to load record: ' + esc(err.message) + "
"; } }); } function ensureCompareIndex() { if (state.compareIndex) return Promise.resolve(state.compareIndex); if (state.compareIndexPromise) return state.compareIndexPromise; var request = fetch("compare/index.json").then(function (r) { if (!r.ok) throw new Error("compare index HTTP " + r.status); return r.json(); }).then(function (index) { state.compareIndex = index; state.compareIndexPromise = null; return index; }, function (error) { state.compareIndexPromise = null; throw error; }); state.compareIndexPromise = request; return request; } function ensureCompareRecord(qid) { var cached = cachedRecord(state.compareRecords, state.compareRecordOrder, qid); if (cached) return Promise.resolve(cached); if (state.compareRecordPromises[qid]) return state.compareRecordPromises[qid]; var request = fetch("compare/records/" + encodeURIComponent(qid) + ".json").then(function (r) { if (!r.ok) throw new Error("compare record HTTP " + r.status); return r.json(); }).then(function (record) { delete state.compareRecordPromises[qid]; storeRecord(state.compareRecords, state.compareRecordOrder, qid, record); return record; }, function (error) { delete state.compareRecordPromises[qid]; throw error; }); state.compareRecordPromises[qid] = request; return request; } function rebuildCompareSelect() { var sel = $("compareQSelect"); sel.innerHTML = ""; state.compareView.forEach(function (item, i) { var option = document.createElement("option"); option.value = String(i); option.textContent = "[" + item.product + "] " + truncate(item.question, 58); sel.appendChild(option); }); sel.value = String(state.compareIdx); } function compareSummaryHtml(index) { return '
Compare scope: all runs joined to the same 815 answerable HERB questions. ' + "Missing or unanswered predictions receive zero.
" + index.runs.map(function (run) { return metricCard(run.label, run.score_pct.toFixed(2) + "%", (run.score_detail || run.correct + "/815") + " Β· " + run.coverage_pct.toFixed(2) + "% coverage (" + run.answered + " answered)", run.slot === "e2e_combined" ? "status-correct" : ""); }).join("") + "
"; } function filterCompare() { var filterSeq = ++state.compareFilterSeq; state.compareLoadSeq++; state.compareView = []; $("compareQSelect").innerHTML = ""; $("compareCounter").textContent = "0 / 0"; $("compareQuestion").innerHTML = '
Loading comparison index…
'; $("compareGrid").innerHTML = ""; ensureCompareIndex().then(function (index) { if (filterSeq !== state.compareFilterSeq || state.mode !== "compare") return; var q = $("compareSearch").value.trim().toLowerCase(); var product = $("compareProduct").value; var typ = $("compareType").value; var filter = $("compareFilter").value; var computationFilter = $("compareComputationFilter").value; state.compareView = index.items.filter(function (item) { if (product && item.product !== product) return false; if (typ && item.type !== typ) return false; if (filter === "disagreement" && !item.disagreement) return false; if (filter === "missing" && !item.any_missing) return false; if (filter === "combined-correct" && !item.only_e2e_combined_correct) return false; if (!computationFilterMatches(item.qid, computationFilter)) return false; if (q && [item.qid, item.gid, item.product, item.question].join(" ").toLowerCase().indexOf(q) === -1) return false; return true; }); var target = sharedIndex(state.compareView); if (target >= 0) state.compareIdx = target; else if (state.compareIdx >= state.compareView.length) state.compareIdx = 0; rebuildCompareSelect(); renderCompare(); }).catch(function (err) { if (filterSeq === state.compareFilterSeq && state.mode === "compare") { $("compareQuestion").innerHTML = '
Failed to load compare index: ' + esc(err.message) + "
"; } }); } function renderCompare() { var seq = ++state.compareLoadSeq; if (state.compareIndex) $("compareSummary").innerHTML = compareSummaryHtml(state.compareIndex); if (!state.compareView.length) { $("compareQuestion").innerHTML = '
No comparisons match.
'; $("compareGrid").innerHTML = ""; $("compareCounter").textContent = "0 / 0"; return; } var item = state.compareView[state.compareIdx]; $("compareQSelect").value = String(state.compareIdx); $("compareCounter").textContent = (state.compareIdx + 1) + " / " + state.compareView.length; $("compareQuestion").innerHTML = '
Loading ' + esc(item.qid) + "…
"; $("compareGrid").innerHTML = ""; ensureCompareRecord(item.qid).then(function (record) { if (seq !== state.compareLoadSeq || state.mode !== "compare") return; rememberQid(record.qid); $("compareQuestion").innerHTML = questionBadges(record, null) + "

" + esc(record.question) + "

" + window.TrajectoryUI.gold(displayGold(record.gold), "Gold") + field("Evidence citations (" + (record.citations || []).length + ")", citationChips(record.product, record.citations)); wireCitationClicks($("compareQuestion")); $("compareGrid").innerHTML = state.compareIndex.runs.map(function (run) { var result = record.runs[run.slot]; var status = runStatus(result); return '
' + '

' + esc(run.label) + '

' + esc(status.label) + "
" + '
' + (result.prediction ? escResolve(result.prediction) : 'No prediction') + "
" + '
Judge + run metadata
' +
         esc(JSON.stringify({
           extracted_answer: result.extracted_answer,
           canonical_score: result.score,
           score_kind: result.score_kind,
           judge_text: result.judge_text,
           confidence: result.confidence,
           failure: result.failure,
           stop_reason: result.stop_reason || result.finish_reason,
           turns: result.turns,
           tokens: result.tokens,
           tool_counts: result.tool_counts,
         }, null, 2)) + "
"; }).join(""); }).catch(function (err) { if (seq === state.compareLoadSeq && state.mode === "compare") { $("compareQuestion").innerHTML = '
Failed to load comparison: ' + esc(err.message) + "
"; } }); } /* ---------------- mode switching ---------------- */ function setMode(mode) { if (mode !== "corpus" && mode !== "eval" && mode !== "compare" && !state.runBySlot[mode]) return; var previousMode = state.mode; var leftStructures = Boolean(window.__e2eStructuresJustClosed); window.__e2eStructuresJustClosed = false; state.mode = mode; Array.prototype.forEach.call($("viewToggle").querySelectorAll("button[data-mode]"), function (button) { button.classList.toggle("active", button.getAttribute("data-mode") === mode); }); $("corpusControls").style.display = mode === "corpus" ? "" : "none"; $("evalControls").style.display = mode === "eval" ? "" : "none"; $("runControls").style.display = state.runBySlot[mode] ? "" : "none"; $("compareControls").style.display = mode === "compare" ? "" : "none"; $("corpusView").style.display = mode === "corpus" ? "flex" : "none"; $("evalView").style.display = mode === "eval" ? "block" : "none"; $("runView").style.display = state.runBySlot[mode] ? "block" : "none"; $("compareView").style.display = mode === "compare" ? "block" : "none"; if (state.runBySlot[mode]) { if (previousMode !== mode || leftStructures) { $("runSearch").value = ""; $("runProduct").value = ""; $("runType").value = ""; $("runStatus").value = ""; $("runComputationFilter").value = ""; } state.runSlot = mode; filterRun(); } else if (mode === "compare") { if (previousMode !== "compare" || leftStructures) { $("compareSearch").value = ""; $("compareProduct").value = ""; $("compareType").value = ""; $("compareFilter").value = ""; $("compareComputationFilter").value = ""; } filterCompare(); } else if (mode === "eval") { if (previousMode !== "eval" || leftStructures) { $("evalSearch").value = ""; $("evalProduct").value = ""; $("kindFilter").value = ""; $("qTypeFilter").value = ""; $("evalComputationFilter").value = ""; } filterEval(); } } /* ---------------- wiring ---------------- */ function wire() { $("viewToggle").addEventListener("click", function (event) { var button = event.target.closest("button[data-mode]"); if (button) setMode(button.getAttribute("data-mode")); }); $("corpusProduct").addEventListener("change", function () { state.curType = null; onProductChange(); }); $("artifactType").addEventListener("change", function () { state.curType = this.value; state.artifactIdx = 0; $("corpusFilter").value = ""; filterArtifacts(); }); $("corpusFilter").addEventListener("input", function () { state.artifactIdx = 0; filterArtifacts(); }); $("artifactSelect").addEventListener("change", function () { state.artifactIdx = parseInt(this.value, 10) || 0; renderArtifact(); }); $("corpusPrevBtn").addEventListener("click", function () { if (state.artifactIdx > 0) { state.artifactIdx--; renderArtifact(); } }); $("corpusNextBtn").addEventListener("click", function () { if (state.artifactIdx < state.artifactsView.length - 1) { state.artifactIdx++; renderArtifact(); } }); $("evalSearchBtn").addEventListener("click", filterEval); $("evalSearch").addEventListener("keydown", function (e) { if (e.key === "Enter") filterEval(); }); $("evalClearBtn").addEventListener("click", function () { $("evalSearch").value = ""; $("evalProduct").value = ""; $("kindFilter").value = ""; $("qTypeFilter").value = ""; $("evalComputationFilter").value = ""; filterEval(); }); $("evalProduct").addEventListener("change", filterEval); $("kindFilter").addEventListener("change", filterEval); $("qTypeFilter").addEventListener("change", filterEval); $("evalComputationFilter").addEventListener("change", filterEval); $("qSelect").addEventListener("change", function () { state.evalIdx = parseInt(this.value, 10) || 0; renderEval(); }); $("evalPrevBtn").addEventListener("click", function () { if (state.evalIdx > 0) { state.evalIdx--; renderEval(); } }); $("evalNextBtn").addEventListener("click", function () { if (state.evalIdx < state.evalView.length - 1) { state.evalIdx++; renderEval(); } }); ["runSearch", "runProduct", "runType", "runStatus", "runComputationFilter"].forEach(function (id) { $(id).addEventListener(id === "runSearch" ? "input" : "change", function () { state.runIdx = 0; filterRun(); }); }); $("runQSelect").addEventListener("change", function () { state.runIdx = parseInt(this.value, 10) || 0; renderRun(); }); $("runPrevBtn").addEventListener("click", function () { if (state.runIdx > 0) { state.runIdx--; renderRun(); } }); $("runNextBtn").addEventListener("click", function () { if (state.runIdx < state.runView.length - 1) { state.runIdx++; renderRun(); } }); ["compareSearch", "compareProduct", "compareType", "compareFilter", "compareComputationFilter"].forEach(function (id) { $(id).addEventListener(id === "compareSearch" ? "input" : "change", function () { state.compareIdx = 0; filterCompare(); }); }); $("compareQSelect").addEventListener("change", function () { state.compareIdx = parseInt(this.value, 10) || 0; renderCompare(); }); $("comparePrevBtn").addEventListener("click", function () { if (state.compareIdx > 0) { state.compareIdx--; renderCompare(); } }); $("compareNextBtn").addEventListener("click", function () { if (state.compareIdx < state.compareView.length - 1) { state.compareIdx++; renderCompare(); } }); document.addEventListener("keydown", function (e) { var tag = (e.target.tagName || "").toLowerCase(); if (tag === "input" || tag === "select" || tag === "textarea") return; if (e.key === "ArrowLeft") { if (state.mode === "corpus") $("corpusPrevBtn").click(); else if (state.mode === "eval") $("evalPrevBtn").click(); else if (state.mode === "compare") $("comparePrevBtn").click(); else $("runPrevBtn").click(); } else if (e.key === "ArrowRight") { if (state.mode === "corpus") $("corpusNextBtn").click(); else if (state.mode === "eval") $("evalNextBtn").click(); else if (state.mode === "compare") $("compareNextBtn").click(); else $("runNextBtn").click(); } }); } /* ---------------- boot ---------------- */ loadAll().then(function () { $("loading").style.display = "none"; var tot = state.products.reduce(function (s, p) { return s + p.n_artifacts; }, 0); $("sidebarFooter").innerHTML = state.products.length + " products Β· " + tot.toLocaleString() + " artifacts Β· " + state.questions.length + " questions
" + 'GitHub Β· ' + 'HF dataset Β· ' + 'arXiv'; populateProductSelects(); buildTabButtons(); wire(); setMode("corpus"); if (state.products.length) { $("corpusProduct").value = state.products[0].name; onProductChange(); } filterEval(); }).catch(function (err) { $("loading").textContent = "Failed to load data: " + err; }); })();