FinCosmos / app.js
YupengCao's picture
Hide the replay button until replay cases are published
23a3bf0 verified
Raw History Blame Contribute Delete
30.1 kB
/* FinCosmos leaderboard. All numbers come from data.js (window.FC), generated by
leaderboard/space_build.py from the E1 records. No numbers are typed here. */
(function () {
"use strict";
const D = window.FC;
const $ = (s, r = document) => r.querySelector(s);
const $$ = (s, r = document) => Array.from(r.querySelectorAll(s));
const esc = (s) => String(s).replace(/[&<>"]/g, (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" }[c]));
const pct = (x) => Math.round(100 * x) + "%";
const f1 = (x) => Number(x).toFixed(1);
const minus = (s) => String(s).replace(/^-/, "−");
const median = (xs) => { const s = [...xs].sort((a, b) => a - b), m = s.length >> 1; return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2; };
const NS = "http://www.w3.org/2000/svg";
document.documentElement.classList.add("js");
/* ---------- order, lookups ---------- */
const isAgent = (a) => a.kind === "agent";
const ranked = D.agents.filter(isAgent).sort((a, b) => b.accepted / b.n - a.accepted / a.n || b.quality - a.quality);
const baselines = D.agents.filter((a) => !isAgent(a));
const ORDER = ranked.concat(baselines);
const byKey = Object.fromEntries(D.agents.map((a) => [a.key, a]));
const vetoName = Object.fromEntries(D.vetoes.map((v) => [v.id, v.name]));
const N = D.sessions_total;
const signedTotal = D.sessions.filter((s) => s.s).length;
const acceptedTotal = D.agents.reduce((t, a) => t + a.accepted, 0);
const arxiv = D.paper.arxiv;
const paperUrl = arxiv ? `https://arxiv.org/abs/${arxiv}` : "#";
const hfUrl = arxiv ? `https://huggingface.co/papers/${arxiv}` : "#";
const dataUrl = "https://huggingface.co/datasets/TheFinAI/FinCosmos";
/* ---------- tooltip ---------- */
const tip = $("#tip");
function showTip(e, html) {
tip.innerHTML = html; tip.classList.add("on");
const r = tip.getBoundingClientRect();
let x = e.clientX + 14, y = e.clientY + 14;
if (x + r.width > innerWidth - 8) x = e.clientX - r.width - 14;
if (y + r.height > innerHeight - 8) y = e.clientY - r.height - 14;
tip.style.left = x + "px"; tip.style.top = y + "px";
}
const hideTip = () => tip.classList.remove("on");
function bindTip(node, html) {
node.addEventListener("mousemove", (e) => showTip(e, typeof html === "function" ? html() : html));
node.addEventListener("mouseleave", hideTip);
}
/* ---------- shared numbers, links ---------- */
const NUM = { sessions: N, accepted: acceptedTotal, signed: signedTotal };
$$("[data-n]").forEach((n) => (n.textContent = NUM[n.dataset.n]));
$$("[data-paper]").forEach((a) => (a.href = paperUrl));
$("#paperLinks").innerHTML = arxiv
? `<a href="${paperUrl}" target="_blank" rel="noreferrer">arXiv ${arxiv} ↗</a><a href="${hfUrl}" target="_blank" rel="noreferrer">Hugging Face Papers ↗</a><a href="${dataUrl}" target="_blank" rel="noreferrer">Scores and tasks ↗</a>`
: "";
$("#footLinks").innerHTML = (arxiv ? `<a href="${paperUrl}" target="_blank" rel="noreferrer">Paper · arXiv ${arxiv} ↗</a><a href="${hfUrl}" target="_blank" rel="noreferrer">Hugging Face Papers ↗</a>` : "")
+ `<a href="${dataUrl}" target="_blank" rel="noreferrer">Scores and tasks ↗</a><span>Evaluator ${esc(D.evaluator)} · updated ${esc(D.updated)}</span>`;
$("#stats").innerHTML = [
[N, "sessions on " + D.tasks_n + " tasks"],
[acceptedTotal, "accepted"],
[signedTotal, "signed off, all vetoed"],
[D.paper.vetoes, "vetoes in " + D.paper.dimensions + " dimensions"],
].map(([b, s]) => `<div class="stat"><b>${b}</b><span>${s}</span></div>`).join("");
/* ---------- rotator ---------- */
(function () {
const r = $("#rotator"), names = ranked.map((a) => a.name);
let i = 0; r.textContent = names[0];
if (matchMedia("(prefers-reduced-motion: reduce)").matches) return;
setInterval(() => {
r.classList.add("off");
setTimeout(() => { i = (i + 1) % names.length; r.textContent = names[i]; r.classList.remove("off"); }, 260);
}, 2400);
})();
/* ---------- orbit map ---------- */
(function () {
const host = $("#orbit"), C = 170, rOf = (q) => 14 + q * 1.06, ACC = 140;
const svg = document.createElementNS(NS, "svg");
svg.setAttribute("viewBox", "-14 -14 368 368");
svg.setAttribute("role", "img");
svg.setAttribute("aria-label", `${N} sessions as stars; distance from the center is quality before vetoes; none reaches the accepted orbit.`);
let g = `<defs><radialGradient id="core"><stop offset="0" stop-color="#2b6cb0" stop-opacity=".16"/><stop offset="1" stop-color="#2b6cb0" stop-opacity="0"/></radialGradient></defs>`;
g += `<circle cx="${C}" cy="${C}" r="${rOf(100)}" fill="url(#core)"/>`;
for (let t = 0; t < D.tasks_n; t++) {
const a = ((t * 36 - 90) * Math.PI) / 180;
g += `<line x1="${C}" y1="${C}" x2="${C + rOf(100) * Math.cos(a)}" y2="${C + rOf(100) * Math.sin(a)}" stroke="rgba(25,60,110,.07)"/>`;
const m = (((t + 0.5) * 36 - 90) * Math.PI) / 180;
g += `<text x="${C + 128 * Math.cos(m)}" y="${C + 128 * Math.sin(m) + 3.5}" text-anchor="middle" font-size="10" fill="#808b9b" font-family="JetBrains Mono, monospace">T${t}</text>`;
}
[25, 50, 75, 100].forEach((q) => {
g += `<circle cx="${C}" cy="${C}" r="${rOf(q)}" fill="none" stroke="rgba(25,60,110,.12)"/>`;
g += `<text x="${C + 3}" y="${C - rOf(q) - 3}" font-size="8.5" fill="#808b9b">${q}</text>`;
});
g += `<circle cx="${C}" cy="${C}" r="${ACC}" fill="none" stroke="#2f855a" stroke-width="1.4" stroke-dasharray="3 4"/>`;
g += `<text x="${C}" y="${C - ACC - 5}" text-anchor="middle" font-size="9.5" fill="#2f855a" font-weight="600">accepted orbit · ${acceptedTotal} sessions</text>`;
svg.innerHTML = g;
const agentIdx = Object.fromEntries(D.agents.map((a, i) => [a.key, i]));
const stars = D.sessions.map((s) => {
const slot = agentIdx[s.a] * 3 + (s.r - 1);
const deg = s.t * 36 + 1.4 + slot * (33.2 / 14) - 90, a = (deg * Math.PI) / 180, r = rOf(s.q);
const x = C + r * Math.cos(a), y = C + r * Math.sin(a);
const grp = document.createElementNS(NS, "g");
grp.setAttribute("class", "star");
const base = byKey[s.a].kind === "agent" ? "#2b6cb0" : "#9aa6b6";
grp.innerHTML = (s.s ? `<circle cx="${x}" cy="${y}" r="7.5" fill="#c9a24a" opacity=".22"/>` : "")
+ `<circle cx="${x}" cy="${y}" r="${s.s ? 3.6 : 2.7}" fill="${s.s ? "#c9a24a" : base}" stroke="#fff" stroke-width="1" opacity="${s.s ? 1 : 0.85}"/>`
+ `<circle class="hit" cx="${x}" cy="${y}" r="8"/>`;
bindTip(grp, () => `<b>${esc(byKey[s.a].name)}</b> · Task ${s.t}, session ${s.r}<br><span class="t-k">Quality before vetoes</span> ${f1(s.q)}<br>`
+ (s.s ? `<span style="color:#e3c376">Signed off by the investor</span><br>` : `<span class="t-k">Ended:</span> ${s.e === "max_turns" ? "turn limit" : "investor walked away"}<br>`)
+ `<span style="color:#f0a37c">${s.v.length} of 15 vetoes fail</span>: ${s.v.join(", ")}`);
svg.appendChild(grp);
return { s, grp };
});
host.appendChild(svg);
const chips = $("#orbitChips"), opts = [{ key: "all", name: "All" }].concat(ORDER);
chips.innerHTML = opts.map((o, i) => `<button class="chip" role="tab" aria-selected="${i === 0}" data-k="${o.key}">${esc(o.name)}</button>`).join("");
function pick(k) {
$$(".chip", chips).forEach((c) => c.setAttribute("aria-selected", c.dataset.k === k));
stars.forEach(({ s, grp }) => grp.classList.toggle("dim", k !== "all" && s.a !== k));
const ss = D.sessions.filter((s) => k === "all" || s.a === k);
const acc = k === "all" ? acceptedTotal : byKey[k].accepted;
$("#orbitSigned").textContent = ss.filter((s) => s.s).length;
$("#orbitAccepted").textContent = acc;
const q = k === "all" ? ss.reduce((t, s) => t + s.q, 0) / ss.length : byKey[k].quality;
$("#orbitEnds").innerHTML = [
[f1(q), "mean quality before vetoes"],
[f1(ss.reduce((t, s) => t + s.v.length, 0) / ss.length), "vetoes failed per session"],
[ss.length, "sessions"],
[ss.filter((s) => s.e === "blocked").length, "investor walked away"],
].map(([b, s]) => `<div><b>${b}</b>${s}</div>`).join("");
}
chips.addEventListener("click", (e) => { const b = e.target.closest(".chip"); if (b) pick(b.dataset.k); });
pick("all");
})();
/* ---------- ticker ---------- */
(function () {
const items = ORDER.map((a, i) => `<span class="tk"><i>${isAgent(a) ? "#" + (i + 1) : "base"}</i><b>${esc(a.name)}</b>quality <u>${f1(a.quality)}</u> · signed off <s>${a.signed}</s> · accepted <q>${a.accepted}</q></span>`).join("");
$("#tickerTrack").innerHTML = items + items + items + items;
})();
/* ---------- replay ---------- */
(function () {
const cases = D.replay || [];
if (!cases.length) { $("#replay").hidden = true; $$('[data-sec="replay"], a[href="#replay"]').forEach((a) => (a.hidden = true)); return; }
const KIND = {
intent: ["Intent", "The strategy is not the one the investor holds."],
data: ["Data", "The reported numbers cannot be rebuilt, or contradict the delivered files."],
time: ["Time", "The backtest uses data after the cutoff, or trades before its signal."],
};
let i = 0, marks = new Set(), done = false, score = [0, 0];
const chk = $("#rChecks");
function render() {
const c = cases[i]; marks = new Set(); done = false;
$("#rKick").textContent = `Task ${c.task} · ${c.persona} · ${c.agent}`;
$("#rN").textContent = `Session ${i + 1} of ${cases.length}`;
$("#rAsk").innerHTML = `<b>What the investor holds (hidden from the agent)</b>${esc(c.ask)}`;
const last = c.chat.length - 1;
$("#rChat").innerHTML = c.chat.map((m, j) => {
const so = j === last && m.who === "investor";
return `<div class="msg msg--${m.who}${so ? " msg--signoff" : ""}"><small>${m.who === "investor" ? "Investor" : esc(c.agent)} · turn ${m.turn}${so ? ' <span class="badge badge--signed">✓ signed off</span>' : ""}</small>${esc(m.text)}</div>`;
}).join("");
const rep = Object.entries(c.reported || {});
$("#rReported").innerHTML = rep.length ? `<span class="rep-k">As reported by the agent</span>` + rep.map(([k, v]) => `<div><b>${esc(minus(v))}</b>${esc(k)}</div>`).join("") : "";
chk.innerHTML = c.checks.map((k) => `<button type="button" class="ck" aria-pressed="false" data-kind="${k.kind}"><span class="ck-ic">?</span><b>${KIND[k.kind][0]}</b><p>${KIND[k.kind][1]}</p></button>`).join("");
$("#rPrompt").hidden = false; $("#rVerdict").hidden = true;
$("#rCheck").hidden = false; $("#rReveal").hidden = false; $("#rNext").hidden = true;
$("#rScore").textContent = score[1] ? `You: ${score[0]} of ${score[1]} right` : "";
}
chk.addEventListener("click", (e) => {
const b = e.target.closest("button.ck"); if (!b || done) return;
const k = b.dataset.kind, on = !marks.has(k);
on ? marks.add(k) : marks.delete(k);
b.setAttribute("aria-pressed", on); $(".ck-ic", b).textContent = on ? "✗" : "?";
});
function open(guessed) {
const c = cases[i]; done = true;
let right = 0;
chk.innerHTML = c.checks.map((k) => {
const marked = marks.has(k.kind), ok = marked === !k.ok; if (ok) right++;
const you = guessed ? `<span class="ck-you ${ok ? "c-pass" : "c-fail"}">${ok ? "you got this" : marked ? "you marked it wrong" : "you missed it"}</span>` : "";
return `<div class="ck ${k.ok ? "is-pass" : "is-fail"}"><span class="ck-ic">${k.ok ? "✓" : "✗"}</span><b>${KIND[k.kind][0]} · ${esc(k.title)}${you}</b><p>${esc(k.detail)}</p>${k.vetoes && k.vetoes.length ? `<span class="ck-v">${k.vetoes.map((v) => esc(v + " " + (vetoName[v] || ""))).join(" · ")}</span>` : ""}</div>`;
}).join("");
if (guessed) { score[0] += right; score[1] += c.checks.length; }
$("#rScore").textContent = score[1] ? `You: ${score[0]} of ${score[1]} right` : "";
$("#rVerdict").innerHTML = `Investor: <span class="badge badge--signed">✓ signed off</span> &nbsp; FinCosmos: <span class="badge badge--fail">✗ rejected</span><br><b>${c.vetoes_failed.length} of 15 vetoes fail</b> (${c.vetoes_failed.map(esc).join(", ")}). Quality before vetoes: <b>${f1(c.quality)}</b>.`;
$("#rVerdict").hidden = false; $("#rPrompt").hidden = true;
$("#rCheck").hidden = true; $("#rReveal").hidden = true; $("#rNext").hidden = false;
$("#rNext").textContent = i === cases.length - 1 ? "Start over →" : "Next session →";
}
$("#rCheck").addEventListener("click", () => open(true));
$("#rReveal").addEventListener("click", () => open(false));
$("#rNext").addEventListener("click", () => { i = (i + 1) % cases.length; if (i === 0) score = [0, 0]; render(); });
render();
})();
/* ---------- leaderboard ---------- */
(function () {
const lerp = (a, b, t) => Math.round(a + (b - a) * t);
const heat = (v, max) => {
const t = Math.max(0, Math.min(1, v / max)), c = [lerp(238, 43, t), lerp(243, 108, t), lerp(249, 176, t)];
return `background:rgb(${c});color:${t > 0.55 ? "#fff" : "#0f1a2a"}`;
};
const CATN = ["Data", "Strategy", "Backtest", "Assess.", "UX"];
const COLS = [
{ k: "rank", h: "#", v: (a) => ORDER.indexOf(a), asc: true },
{ k: "agent", h: "Agent", v: (a) => a.name, asc: true },
{ k: "accepted", h: "Accepted", v: (a) => a.accepted / a.n },
{ k: "signed", h: "Signed off", v: (a) => a.signed / a.n },
{ k: "quality", h: "Quality", v: (a) => a.quality },
...CATN.map((h, j) => ({ k: "c" + j, h, v: (a) => a.cats[j] })),
{ k: "vetoes", h: "Vetoes", v: (a) => a.vetoes_mean, asc: true },
{ k: "asked", h: "Asked", v: (a) => a.coverage },
{ k: "usd", h: "Cost", v: (a) => a.usd, asc: true },
];
let sortK = "rank", asc = true, openKey = null;
const lb = $("#lb");
function cells(a) {
const rk = isAgent(a) ? ranked.indexOf(a) + 1 : "—";
return `<td class="rk">${rk}</td>`
+ `<td class="ag"><b>${esc(a.name)}<span class="chev" aria-hidden="true">›</span></b><span>${esc(a.model)}</span></td>`
+ `<td><span class="zero">${pct(a.accepted / a.n)}</span></td>`
+ `<td><span class="sg">${pct(a.signed / a.n)}</span></td>`
+ `<td><span class="heat" style="${heat(a.quality, 60)}">${f1(a.quality)}</span><span class="ci">[${a.ci.map(f1).join(", ")}]</span></td>`
+ a.cats.map((c) => `<td><span class="heat" style="${heat(c, 60)}">${f1(c)}</span></td>`).join("")
+ `<td>${f1(a.vetoes_mean)}</td><td>${pct(a.coverage)}</td><td>$${a.usd.toFixed(2)}</td>`;
}
function detail(a) {
const ss = D.sessions.filter((s) => s.a === a.key);
const grid = D.tasks.map((t) => {
const ts = ss.filter((s) => s.t === t.seed).sort((x, y) => x.r - y.r);
return `<div class="tcol"><span>T${t.seed}</span>${ts.map((s) => `<div class="tcell${s.s ? " sat" : ""}" style="${heat(s.q, 60)}" data-tip="${esc(`<b>Task ${t.seed}</b>, session ${s.r} · ${t.persona}, difficulty ${t.difficulty}<br>Quality ${f1(s.q)} · ${s.v.length} vetoes fail${s.s ? '<br><span style="color:#e3c376">Signed off</span>' : ""}`)}"></div>`).join("")}</div>`;
}).join("");
const vb = D.vetoes.map((v) => `<span>${v.id} ${esc(v.name)}</span><div class="vbar"><i style="width:${100 * a.veto[v.id]}%"></i></div><em>${pct(a.veto[v.id])}</em>`).join("");
const pl = [["Turns per session", f1(a.turns)], ["Turns cut at 300 s", pct(a.cut)], ["Minutes per session", f1(a.minutes)], ["Input tokens (M)", a.tokens_m.toFixed(2)], ["Investor's details stated by the end", `${f1(a.stated)} of ${f1(a.hidden)}`], ["Cost per session", "$" + a.usd.toFixed(2)]]
.map(([k, v]) => `<span>${k}</span><b>${v}</b>`).join("");
return `<div class="det"><div class="card"><h4>Every session, by task</h4><div class="tgrid">${grid}</div><div class="tkey"><span><i style="background:#2b6cb0"></i>quality before vetoes, darker is higher</span><span><i style="box-shadow:inset 0 0 0 2px #c9a24a"></i>signed off by the investor</span></div></div>`
+ `<div class="card"><h4>Share of sessions failing each veto</h4><div class="vbars">${vb}</div></div>`
+ `<div class="card"><h4>Process</h4><div class="plist">${pl}</div></div></div>`;
}
function render() {
const col = COLS.find((c) => c.k === sortK);
const rows = [...ORDER].sort((x, y) => { const a = col.v(x), b = col.v(y); const d = typeof a === "string" ? a.localeCompare(b) : a - b; return asc ? d : -d; });
lb.innerHTML = `<thead><tr>${COLS.map((c) => `<th data-k="${c.k}" class="${c.k === sortK ? "sorted" + (asc ? " asc" : "") : ""}">${c.h}</th>`).join("")}</tr></thead><tbody>`
+ rows.map((a) => `<tr class="row${isAgent(a) ? "" : " base"}${openKey === a.key ? " open" : ""}" data-k="${a.key}">${cells(a)}</tr>` + (openKey === a.key ? `<tr class="detail"><td colspan="${COLS.length}">${detail(a)}</td></tr>` : "")).join("") + "</tbody>";
$$(".tcell", lb).forEach((n) => bindTip(n, n.dataset.tip));
fit();
}
// The table scrolls sideways on narrow screens; keep the open row's breakdown in view.
function fit() { const d = $(".det", lb); if (d) d.style.width = lb.parentElement.clientWidth + "px"; }
addEventListener("resize", fit);
lb.addEventListener("click", (e) => {
const th = e.target.closest("th");
if (th) { const c = COLS.find((x) => x.k === th.dataset.k); if (sortK === c.k) asc = !asc; else { sortK = c.k; asc = !!c.asc; } render(); return; }
const tr = e.target.closest("tr.row");
if (tr) { openKey = openKey === tr.dataset.k ? null : tr.dataset.k; render(); }
});
render();
})();
/* ---------- graders ---------- */
(function () {
const G = [
["The investor signs off", "signed", "count as a success when the investor's sign-off is the grade"],
["+ LLM-judged vetoes", "data", "still pass when an LLM judge also checks its four vetoes (2.7, 4.1, 5.1, 5.6)"],
["All 15 vetoes · FinCosmos", "pass", "pass once programs also check the delivery itself"],
];
const COLOR = { signed: "#c9a24a", data: "#2b6cb0", pass: "#2f855a" };
const seg = $("#graderSeg");
seg.innerHTML = G.map((g, i) => `<button role="tab" aria-selected="${i === 0}" data-i="${i}">${g[0]}</button>`).join("");
const bars = $("#grBars");
bars.innerHTML = ORDER.map((a) => `<div class="hrow${isAgent(a) ? "" : " muted"}" data-k="${a.key}"><span>${esc(a.name)}</span><div class="htrack"><i></i></div><em></em></div>`).join("");
function pick(i) {
$$("button", seg).forEach((b) => b.setAttribute("aria-selected", +b.dataset.i === i));
const total = D.agents.reduce((t, a) => t + a.graders[i], 0);
$("#grTotal").textContent = total; $("#grTotal").style.color = i === 2 ? "#c05621" : "#0f1a2a";
$("#grTotalNote").textContent = `of ${N} sessions ${G[i][2]}.`;
$$(".hrow", bars).forEach((r) => {
const a = byKey[r.dataset.k], v = a.graders[i];
$("i", r).style.width = (100 * v) / a.n + "%"; $("i", r).style.background = isAgent(a) ? COLOR[G[i][1]] : "";
$("em", r).textContent = `${v} / ${a.n}`;
});
}
seg.addEventListener("click", (e) => { const b = e.target.closest("button"); if (b) pick(+b.dataset.i); });
pick(0);
const sat = D.sessions.filter((s) => s.s);
const groups = [
["Audit and process", ["3.7", "3.8"]], ["Recomputation", ["3.2"]], ["LLM-judged", ["4.1", "5.1", "5.6"]],
["Point-in-time", ["1.1", "3.1"]], ["Strategy fidelity", ["2.1", "2.2", "2.3", "2.4", "2.5", "2.6", "2.7"]],
].map(([n, ids]) => [n, ids, sat.filter((s) => s.v.some((v) => ids.includes(v))).length]).sort((a, b) => b[2] - a[2]);
$("#grWhy").innerHTML = groups.map(([n, ids, c]) => `<div class="hrow" title="${ids.join(", ")}"><span>${n}</span><div class="htrack"><i style="width:${(100 * c) / sat.length}%"></i></div><em>${pct(c / sat.length)}</em></div>`).join("");
$$("#grWhy .hrow").forEach((r, j) => bindTip(r, `<b>${groups[j][0]}</b> (${groups[j][1].join(", ")})<br>${groups[j][2]} of ${sat.length} signed-off sessions fail at least one`));
})();
/* ---------- workflow funnel ---------- */
(function () {
const STEPS = [
["The investor's strategy", "The delivered strategy is the one the investor holds", "2.1–2.7"],
["Data within the cutoff", "No delivered row is dated after the cutoff", "1.1"],
["No look-ahead", "Every fill comes after its signal", "3.1"],
["Reproducible", "Delivered trades reproduce the reported returns", "3.2 ≥ 2"],
];
const agentsN = ranked.reduce((t, a) => t + a.n, 0);
const cell = (v, n, base) => `<div class="fcell${v === 0 ? " zero" : ""}${base ? " base" : ""}"><div class="htrack"><i style="width:${(100 * v) / n}%"></i></div><em>${v}</em></div>`;
$("#funnel").innerHTML = `<thead><tr><th>Step</th>${ORDER.map((a) => `<th>${esc(a.name)}</th>`).join("")}<th>In sequence · ${agentsN} agent sessions</th></tr></thead><tbody>`
+ STEPS.map((s, k) => `<tr><td class="st"><b><i>(${k + 1})</i>${s[0]}</b><span>${s[1]} · ${s[2]}</span></td>${ORDER.map((a) => `<td class="${isAgent(a) ? "" : "base"}">${cell(a.funnel[k], a.n, !isAgent(a))}</td>`).join("")}<td class="seqc">${cell(D.seq[k], agentsN)}</td></tr>`).join("") + "</tbody>";
})();
/* ---------- elicitation ---------- */
(function () {
const row = (a, v, sub) => `<div class="hrow${isAgent(a) ? "" : " muted"}"><span>${esc(a.name)}</span><div class="htrack"><i style="width:${100 * v}%"></i></div><em>${pct(v)}</em></div>`;
$("#elCov").innerHTML = ORDER.map((a) => row(a, a.coverage)).join("");
$("#elAssume").innerHTML = ORDER.filter((a) => a.assume != null).map((a) => row(a, a.assume)).join("");
})();
/* ---------- learning ---------- */
(function () {
const L = D.learning, n = L.map((t) => t.sessions);
const lastCount = {}; L.forEach((t) => (lastCount[t.last_veto] = (lastCount[t.last_veto] || 0) + 1));
const [topV, topN] = Object.entries(lastCount).sort((a, b) => b[1] - a[1])[0];
const mFirst = median(L.map((t) => t.first)), mF5 = median(L.map((t) => t.first5)), mL5 = median(L.map((t) => t.last5));
$("#learnStats").innerHTML = [
[`${L.length} / ${L.length}`, "tasks learned, none passed at the first attempt"],
[median(n), "median sessions to the first acceptance"],
[n.reduce((a, b) => a + b, 0), `sessions in all, ${Math.min(...n)} to ${Math.max(...n)} per task`],
[`${f1(mFirst)} → ${f1(mL5)}`, "median failed vetoes, first session → last five"],
].map(([b, s]) => `<div><b>${b}</b>${s}</div>`).join("")
+ `<p class="wide">The last veto to clear is most often <b>${topV} ${esc(vetoName[topV])}</b>, in ${topN} of ${L.length} tasks. The leak-free rule withholds its reasons, so the agent has to infer them from the investor's words.</p>`;
const W = 640, H = 300, P = { l: 40, r: 70, t: 14, b: 34 }, X = ["1st session", "First 5", "Last 5", "Accepted"];
const ymax = Math.max(15, ...L.map((t) => t.first));
const x = (i) => P.l + (i * (W - P.l - P.r)) / 3, y = (v) => P.t + (1 - v / ymax) * (H - P.t - P.b);
const pts = (vals) => vals.map((v, i) => `${x(i)},${y(v)}`).join(" ");
let s = `<g class="grid">${[0, 3, 6, 9, 12, 15].map((v) => `<line x1="${P.l}" x2="${W - P.r}" y1="${y(v)}" y2="${y(v)}"/>`).join("")}</g>`;
s += `<g class="axis">${[0, 3, 6, 9, 12, 15].map((v) => `<text x="${P.l - 10}" y="${y(v) + 4}" text-anchor="end">${v}</text>`).join("")}${X.map((t, i) => `<text x="${x(i)}" y="${H - 10}" text-anchor="middle">${t}</text>`).join("")}</g>`;
s += `<text x="12" y="${(H - P.b + P.t) / 2}" transform="rotate(-90 12 ${(H - P.b + P.t) / 2})" text-anchor="middle" font-size="11" fill="#808b9b">Failed vetoes per session</text>`;
L.forEach((t, j) => {
const v = [t.first, t.first5, t.last5, 0];
s += `<g class="tl" data-j="${j}"><polyline points="${pts(v)}" fill="none" stroke="#b8c2cf" stroke-width="1.6"/>${v.map((u, i) => `<circle cx="${x(i)}" cy="${y(u)}" r="2.4" fill="#b8c2cf"/>`).join("")}<polyline points="${pts(v)}" fill="none" stroke="transparent" stroke-width="12"/></g>`;
});
const mv = [mFirst, mF5, mL5, 0];
s += `<polyline points="${pts(mv)}" fill="none" stroke="#2b6cb0" stroke-width="3" stroke-linejoin="round"/>`;
s += mv.map((u, i) => `<circle cx="${x(i)}" cy="${y(u)}" r="5" fill="#2b6cb0" stroke="#fff" stroke-width="2"/>`).join("");
s += mv.slice(0, 3).map((u, i) => `<text x="${x(i) + 10}" y="${y(u) - 9}" text-anchor="start" font-size="12" font-weight="600" fill="#2b6cb0" stroke="#fff" stroke-width="3" paint-order="stroke">${f1(u)}</text>`).join("");
s += `<text x="${x(3) + 10}" y="${y(0) - 8}" font-size="12" font-weight="600" fill="#2f855a">0 · accepted</text>`;
const host = $("#learnChart");
host.innerHTML = `<svg viewBox="0 0 ${W} ${H}" role="img" aria-label="Failed vetoes per session along practice, for each of the ten tasks and their median">${s}</svg>`;
$$(".tl", host).forEach((g) => {
const t = L[+g.dataset.j];
g.addEventListener("mouseenter", () => $$("polyline,circle", g).forEach((p) => { if (p.getAttribute("stroke") !== "transparent") { p.setAttribute(p.tagName === "circle" ? "fill" : "stroke", "#4a576a"); } }));
g.addEventListener("mouseleave", () => $$("polyline,circle", g).forEach((p) => { if (p.getAttribute("stroke") !== "transparent") { p.setAttribute(p.tagName === "circle" ? "fill" : "stroke", "#b8c2cf"); } }));
bindTip(g, `<b>Difficulty ${t.difficulty}</b> · ${t.sessions} sessions to acceptance<br><span class="t-k">Failed vetoes</span> ${t.first} first → ${f1(t.first5)} first five → ${f1(t.last5)} last five<br><span class="t-k">Investor signed off in</span> ${pct(t.satisfied)} of practice sessions<br><span class="t-k">Last to clear</span> ${t.last_veto} ${esc(vetoName[t.last_veto] || "")}`);
});
})();
/* ---------- benchmark ---------- */
(function () {
const P = D.paper;
$("#parts").innerHTML = [
["Strategy Generator", "A hidden strategy per task", `A strategy fixes one option for each of six components: stock selection, market timing, portfolio construction, risk control, execution, and transaction cost. New tasks can be drawn whenever a released set saturates.`,
[`<b>~${P.strategies}</b> strategies before parameters`, "<b>6</b> components", "Seeded and reproducible"]],
["User Simulator", "An investor who answers, never volunteers", `A language model plays the investor. It opens with a vague request, states a detail only when the agent asks about it or proposes it, and objects when a proposal conflicts with what it wants.`,
[`<b>${P.personas}</b> personas`, "Never says “point-in-time” or “cutoff”", "Signs off only with cited evidence"]],
["Evaluator", "The delivery, not the conversation", `Programs recompute numbers and check dates, an LLM judge grades only semantics, and every score carries its evidence. Feedback is leak-free: it never reveals the hidden strategy.`,
[`<b>${P.dimensions}</b> dimensions, <b>${P.vetoes}</b> vetoes`, "No reference answer needed", "Every score tied to an evaluator version"]],
["Setup", "Frozen, so scores compare", `The released benchmark freezes the first ten sampled tasks (seeds 0 to 9) on U.S. equities. The agent receives no data: it searches the Web and calls ${P.fmp_apis} financial APIs, then must stay before each task's cutoff.`,
[`<b>${D.tasks_n}</b> tasks × <b>${D.sessions_per_task}</b> sessions`, `<b>${P.turns}</b> turns of <b>${P.seconds_per_turn}</b> s`, "One data cutoff per task"]],
].map(([h, k, p, li]) => `<div><span class="m-k">${k}</span><h3>${h}</h3><p>${p}</p><ul>${li.map((x) => `<li>${x}</li>`).join("")}</ul></div>`).join("");
const J = { program: "program", LLM: "LLM judge", "program + LLM": "program + LLM" };
$("#vgrid").innerHTML = D.vetoes.map((v) => `<div class="vc"><span class="vc-id">${v.id} · ${esc(v.cat)}<span class="jb">${J[v.judge] || esc(v.judge)}</span></span><h4>${esc(v.name)}</h4><div class="vc-n"><b class="${v.sat_fail ? "" : "z"}">${v.sat_fail}</b>of ${signedTotal}</div><p>${esc(v.text)}</p></div>`).join("");
const H = D.human;
$("#trust").innerHTML = [
[pct(H.agent / H.sessions), `of failed sessions are the agent's fault, according to two financial experts (${H.agent} of ${H.sessions} sampled; ${H.simulator} the simulator, ${H.evaluator} the evaluator).`],
[`κ = ${H.kappa_judge}`, `agreement between the LLM judge and the experts on the judge-graded vetoes; the experts agree with each other at ${H.kappa_experts}.`],
[`${acceptedTotal} of ${N}`, `sessions accepted. Every session the investor signed off fails at least one program check.`],
].map(([b, p]) => `<div><b>${b}</b><p>${p}</p></div>`).join("");
})();
/* ---------- chrome: reveal, header, progress, nav ---------- */
const io = new IntersectionObserver((es) => es.forEach((e) => { if (e.isIntersecting) { e.target.classList.add("in"); io.unobserve(e.target); } }), { threshold: 0.08 });
$$(".reveal").forEach((n) => io.observe(n));
const header = $("#header"), prog = $("#progress");
const onScroll = () => {
header.classList.toggle("scrolled", scrollY > 10);
const h = document.documentElement.scrollHeight - innerHeight;
prog.style.transform = `scaleX(${h > 0 ? scrollY / h : 0})`;
};
addEventListener("scroll", onScroll, { passive: true }); onScroll();
const links = $$("#navLinks a");
const so = new IntersectionObserver((es) => es.forEach((e) => {
if (e.isIntersecting) links.forEach((a) => a.classList.toggle("active", a.dataset.sec === e.target.id));
}), { rootMargin: "-45% 0px -50% 0px" });
links.forEach((a) => { const s = document.getElementById(a.dataset.sec); if (s) so.observe(s); });
})();