Spaces:
Running
Running
| // One task: what is forecast, how systems score, run days, inputs, case studies, files. | |
| // Every task renders from the same template; nothing here knows a particular target. | |
| import { anchorStrip, fanChart, historyChart, horizonChart } from "./charts.js"; | |
| import { fileUrl, load } from "./data.js"; | |
| import { | |
| bySystem, dayLabel, empty, fmt, h, leaderboard, md, more, redraws, sectionHead, segmented, | |
| sourceLink, system, systemCell, timeIn, | |
| } from "./util.js"; | |
| const MODES = [["live", "Live"], ["mock", "Offline replay"]]; | |
| const RUN_DAYS_SHOWN = 10; | |
| export function areaName(index, key) { | |
| const [domainId, leafId] = (key || "").split("/"); | |
| const domain = index.taxonomy.domains.find((d) => d.id === domainId); | |
| const leaf = domain?.leaves.find((l) => l.id === leafId); | |
| return { domain, leaf }; | |
| } | |
| function pips(done, total) { | |
| return h("span", { class: "pips", "aria-hidden": "true" }, d3.range(total).map((i) => h("i", { class: i < done ? "on" : null }))); | |
| } | |
| function head(index, t) { | |
| const s = t.spec; | |
| const { domain, leaf } = areaName(index, s.taxonomy); | |
| const tz = s.schedule.timezone; | |
| const cadence = s.schedule.every_days === 1 ? "every day" : `every ${s.schedule.every_days} days`; | |
| const zone = new Intl.DateTimeFormat("en-US", { timeZone: tz, timeZoneName: "short" }) | |
| .formatToParts(new Date(t.status.next_run)).find((p) => p.type === "timeZoneName")?.value; | |
| const fact = (label, value) => h("span", {}, h("b", {}, label), value); | |
| const sections = [["target", "Target"], ["results", "Results"], ["run-days", "Run days"], ["inputs", "Inputs"], | |
| ...(t.case_studies.length ? [["cases", "Case studies"]] : []), ["files", "Task files"]]; | |
| return h("header", { class: "page-head" }, | |
| h("div", { class: "crumbs" }, h("a", { href: "#/tasks" }, "Tasks"), "›", | |
| domain ? h("a", { href: `#/tasks?area=${domain.id}` }, domain.name) : null, leaf ? "›" : null, | |
| leaf ? h("a", { href: `#/tasks?area=${s.taxonomy}` }, leaf.name) : null), | |
| h("h1", {}, s.title), | |
| h("p", { class: "lead" }, s.question), | |
| h("div", { class: "facts" }, | |
| fact("Region", s.region), fact("Unit", s.unit), | |
| fact("Horizon", `${d3.min(s.horizons)}–${d3.max(s.horizons)} days ahead`), | |
| fact("Forecast", `${s.quantiles.length} quantiles`), | |
| fact("Due", `${s.schedule.cutoff} ${zone}, ${cadence}`), | |
| fact("Live since", `${dayLabel(t.status.live_since, "day")} · ${t.status.run_days} run days`), | |
| fact("Next run", timeIn(t.status.next_run, tz, true))), | |
| h("nav", { class: "anchors", "aria-label": "Sections" }, sections.map(([id, text]) => | |
| h("a", { href: `#${id}`, onclick: (e) => { e.preventDefault(); document.getElementById(id)?.scrollIntoView({ behavior: "smooth" }); } }, text)))); | |
| } | |
| function targetSection(t) { | |
| const s = t.spec; | |
| const chart = h("div", { class: "chart" }); | |
| redraws.push(() => historyChart(chart, { truth: t.truth, runDays: t.run_days.map((d) => d.run_date), unit: s.unit, title: s.title })); | |
| return h("section", { class: "section" }, | |
| sectionHead("target", "What is forecast"), | |
| h("div", { class: "card split" }, | |
| h("div", { class: "target-text" }, | |
| h("p", { class: "def-main" }, s.summary), | |
| h("dl", { class: "def" }, | |
| h("dt", {}, "Definition"), h("dd", {}, s.y.definition), | |
| h("dt", {}, "Settled on"), h("dd", {}, s.y.settlement), | |
| h("dt", {}, "Source"), h("dd", {}, sourceLink(s.y.source)))), | |
| h("div", {}, chart, h("p", { class: "note" }, `First published values, last ${t.truth.length} days; blue ticks mark run days.`)))); | |
| } | |
| function resultsSection(t, state, rerender) { | |
| const rows = t.leaderboard[state.mode] || []; | |
| const n = rows[0]?.n; | |
| const horizon = h("div", { class: "chart" }); | |
| const byHorizon = t.by_horizon[state.mode] || []; | |
| const order = [...new Set(byHorizon.map((r) => r.system))].sort(bySystem); | |
| if (byHorizon.length) redraws.push(() => horizonChart(horizon, byHorizon, order)); | |
| return h("section", { class: "section" }, | |
| sectionHead("results", "Results", | |
| state.mode === "live" | |
| ? "Live forecasts, scored once the values are published. Every row uses the same (run day, target date) pairs; 1 is the seasonal naive forecast, lower is better." | |
| : "Agents replaying each run day offline from the materials the live agents saved, next to the same TSFM and expert forecasts.", | |
| segmented(MODES, state.mode, (mode) => { state.mode = mode; rerender(); }, "Mode")), | |
| h("div", { class: "card" }, | |
| leaderboard(rows, t.spec.unit), | |
| n ? h("p", { class: "note", style: { marginTop: "10px" } }, `${n} common (run day, target date) pairs.`) : null, | |
| byHorizon.length ? more("By horizon", horizon, h("p", { class: "note" }, "Later horizons resolve later, so their points rest on fewer run days.")) : null), | |
| state.mode === "live" ? continualCard(t) : null); | |
| } | |
| /** Self-improvement: each continual agent against the fresh agent of the same harness and | |
| * model, on the live pairs both resolved. */ | |
| function continualCard(t) { | |
| const pairs = (t.continual || []).filter((p) => p.rows.length === 2); | |
| if (!t.continual?.length) return null; | |
| const score = (rows, id) => rows.find((r) => r.system === id); | |
| return h("div", { class: "card" }, | |
| h("div", { class: "card-title" }, "Self-improvement: continual versus fresh"), | |
| h("p", { class: "secondary small", style: { marginBottom: "10px" } }, | |
| "A continual agent continues, each run day, from a snapshot of the sandbox and native session of its previous run day, sees its earlier submissions and is asked to improve. It runs beside the fresh agent of the same harness and model; on the same pairs, the difference is what carried-over experience adds."), | |
| pairs.length | |
| ? h("div", { class: "table-wrap" }, h("table", {}, | |
| h("thead", {}, h("tr", {}, h("th", {}, "Agent"), h("th", { class: "num" }, "Fresh"), h("th", { class: "num" }, "Continual"), | |
| h("th", { class: "num" }, "Difference"), h("th", { class: "num" }, "Pairs"))), | |
| h("tbody", {}, pairs.map((p) => { | |
| const fresh = score(p.rows, p.fresh), cont = score(p.rows, p.continual); | |
| const diff = cont.scaled_crps - fresh.scaled_crps; | |
| return h("tr", {}, h("td", {}, systemCell(p.fresh)), | |
| h("td", { class: "num" }, fmt.ratio(fresh.scaled_crps)), h("td", { class: "num" }, fmt.ratio(cont.scaled_crps)), | |
| h("td", { class: "num", title: "continual minus fresh scaled CRPS; negative means the continual agent did better" }, | |
| `${diff > 0 ? "+" : ""}${fmt.ratio(diff)}`), | |
| h("td", { class: "num" }, String(fresh.n))); | |
| })))) | |
| : h("p", { class: "note" }, "No continual forecast has been scored yet.")); | |
| } | |
| /** The forecasts a mode shows: live, or the replaying agents next to the live non-agents. */ | |
| export function inMode(forecasts, mode) { | |
| return forecasts.filter((f) => (mode === "live" ? f.mode === "live" : f.mode === "mock" || system(f.system).kind !== "agent")) | |
| .sort((a, b) => bySystem(a.system, b.system)); | |
| } | |
| function runDaysSection(t, state) { | |
| const shown = state.allDays ? t.run_days : t.run_days.slice(0, RUN_DAYS_SHOWN); | |
| // Columns: every non-reference system the mode shows (replaying agents plus the live others). | |
| const columns = [...new Set(t.run_days.flatMap((d) => [...d.submitted[state.mode], ...d.submitted.live.filter((id) => system(id).kind !== "agent")]))] | |
| .filter((id) => system(id).kind !== "reference" && (state.mode === "live" || system(id).kind !== "agent" || t.run_days.some((d) => d.submitted.mock.includes(id)))) | |
| .sort(bySystem); | |
| const latest = t.run_days[0]; | |
| const chart = h("div", { class: "chart" }); | |
| const legend = h("div", { class: "legend" }); | |
| if (latest) { | |
| load(`tasks/${t.id}/${latest.run_date}.json`).then((day) => { | |
| const forecasts = inMode(day.forecasts, state.mode); | |
| const focus = forecasts.find((f) => system(f.system).kind === "agent")?.system; | |
| const draw = () => fanChart(chart, { title: t.spec.title, unit: t.spec.unit, runDate: day.run_date, truth: day.truth, forecasts, | |
| lookback: 21, visible: new Set(forecasts.map((f) => f.system)), focus, height: 320 }); | |
| legend.replaceChildren(...forecasts.map((f) => h("span", { class: "chip" }, systemCell(f.system)))); | |
| redraws.push(draw); | |
| draw(); | |
| }); | |
| } | |
| const table = h("div", { class: "table-wrap" }, h("table", {}, | |
| h("thead", {}, h("tr", {}, h("th", {}, "Run day"), h("th", {}, "Resolved"), | |
| columns.map((id) => h("th", { class: "num wrap-head", title: system(id).label }, system(id).label)), | |
| h("th", { class: "num" }, "Materials"))), | |
| h("tbody", {}, shown.map((d, i) => { | |
| const scores = { ...d.scores.live, ...(state.mode === "mock" ? d.scores.mock : {}) }; | |
| if (state.mode === "mock") for (const id of Object.keys(d.scores.live)) if (system(id).kind === "agent" && !(id in d.scores.mock)) delete scores[id]; | |
| const values = columns.map((id) => scores[id]?.scaled_crps); | |
| const best = d3.min(values.filter((v, i) => v != null && system(columns[i]).kind !== "reference")); | |
| const files = d3.sum(Object.values(d.materials), (m) => m.files); | |
| const open = () => { location.hash = `#/t/${t.id}/${d.run_date}${state.mode === "mock" ? "?mode=mock" : ""}`; }; | |
| return [h("tr", { class: "link", onclick: open }, | |
| h("td", { class: "nowrap" }, h("a", { href: `#/t/${t.id}/${d.run_date}`, onclick: (e) => e.stopPropagation() }, dayLabel(d.run_date, "dow"))), | |
| h("td", { class: "nowrap" }, pips(d.resolved, d.target_dates), h("span", { class: "small secondary" }, `${d.resolved}/${d.target_dates}`)), | |
| columns.map((id, i) => { | |
| const submitted = (system(id).kind === "agent" ? d.submitted[state.mode] : d.submitted.live).includes(id); | |
| return h("td", { class: `num${values[i] != null && values[i] === best ? " best" : ""}` }, | |
| values[i] != null ? fmt.ratio(values[i]) : h("span", { class: "muted" }, submitted ? "pending" : "—")); | |
| }), | |
| h("td", { class: "num" }, files ? String(files) : "—"))]; | |
| })))); | |
| return h("section", { class: "section" }, | |
| sectionHead("run-days", "Run days", | |
| `Scaled CRPS of each system on each run day, over that day's resolved target dates (${state.mode === "live" ? "live" : "offline replay"}). Open a day for its forecasts, agent runs and materials.`), | |
| latest ? h("div", { class: "card" }, | |
| h("div", { class: "card-title" }, `Latest forecasts · made ${dayLabel(latest.run_date, "long")}`), | |
| legend, chart, | |
| h("p", { class: "note" }, "Ink: published values. Lines: each system's median; band: the first agent's 10–90% interval. Open the run day to compare systems one by one.")) : null, | |
| h("div", { class: "card" }, t.run_days.length ? table : empty("No run day yet."), | |
| t.run_days.length > RUN_DAYS_SHOWN | |
| ? h("div", { style: { marginTop: "10px" } }, h("button", { type: "button", class: "link-button", onclick: () => { state.allDays = !state.allDays; state.rerender(); } }, | |
| state.allDays ? "Show fewer" : `Show all ${t.run_days.length} run days`)) | |
| : null)); | |
| } | |
| function inputsSection(t) { | |
| const s = t.spec; | |
| const b = s.baseline; | |
| const x = s.x; | |
| const latest = t.status.latest_run; | |
| const notes = t.latest_x?.covariate_notes || {}; | |
| const noteRows = Object.entries(notes).map(([key, value]) => | |
| [h("dt", {}, key), h("dd", {}, typeof value === "string" ? value : Object.entries(value).map(([k, v]) => `${k.replaceAll("_", " ")}: ${v}`).join(" · "))]); | |
| const files = ["series.csv", "task.json", "MANIFEST.json"]; | |
| return h("section", { class: "section" }, | |
| sectionHead("inputs", "Inputs", "What the comparison systems use. Agents receive none of it: only the instruction, the run date and the open internet."), | |
| h("div", { class: "grid-2" }, | |
| b ? h("div", { class: "card" }, | |
| h("div", { class: "eyebrow" }, b.reference ? "Reference forecast" : "Expert forecast"), | |
| h("h3", { style: { fontSize: "16px", margin: "6px 0 8px" } }, b.name), | |
| h("p", { class: "secondary small" }, b.method), | |
| h("dl", { class: "def", style: { marginTop: "12px" } }, | |
| h("dt", {}, "Form"), h("dd", {}, b.kind === "point" ? "Point forecast (scored by absolute error)" : "Quantile forecast"), | |
| h("dt", {}, "Agents see it"), h("dd", {}, b.visible_to_agents ? "Yes: it is published while the agents run" : "No: it is published after the cutoff"), | |
| h("dt", {}, "Source"), h("dd", {}, sourceLink(b.source)))) | |
| : h("div", { class: "card" }, | |
| h("div", { class: "eyebrow" }, "Expert forecast"), | |
| h("p", { class: "secondary small", style: { margin: "6px 0 0" } }, "No public expert or official forecast of this series exists, so systems are compared with each other and with the seasonal naive reference.")), | |
| h("div", { class: "card" }, | |
| h("div", { class: "eyebrow" }, "Curated covariates · input to the TSFM baselines"), | |
| h("p", { class: "secondary small", style: { margin: "6px 0 10px" } }, | |
| md(`Chosen by the task's author and snapshotted before each run ($X_d$). History from ${dayLabel(x.history_start, "long")}.`)), | |
| h("div", { class: "covariates" }, x.groups.map((g) => h("div", { class: "covariate" }, | |
| h("h4", {}, g.name), | |
| g.what ? h("p", { class: "what" }, g.what) : null, | |
| g.why ? h("p", { class: "why" }, g.why) : null, | |
| h("p", { class: "src" }, g.url ? h("a", { href: g.url, target: "_blank", rel: "noopener" }, g.source) : g.source)))), | |
| more("Columns, notes and downloads", | |
| h("dl", { class: "def" }, | |
| h("dt", {}, "Columns"), h("dd", {}, h("div", { class: "chips" }, x.groups.flatMap((g) => g.columns.map((c) => h("code", {}, c))))), | |
| ...noteRows.flat(), | |
| h("dt", {}, md("Also in $X_d$")), h("dd", {}, x.raw.join("; "))), | |
| latest ? h("div", { class: "downloads", style: { marginTop: "10px" } }, | |
| h("span", { class: "note" }, md(`$X_d$ of ${dayLabel(latest, "day")}:`)), | |
| files.map((name) => h("a", { href: fileUrl(`runs/${t.id}/${latest}/x/${name}`), target: "_blank", rel: "noopener" }, `${name} ↓`))) : null)))); | |
| } | |
| const STEP_KIND = { model: "Model", reference: "Reference", combine: "Combine", uncertainty: "Uncertainty" }; | |
| function caseBody(t, cs, day) { | |
| const value = (f, date) => f.values.find((v) => v.target_date === date); | |
| const actualOf = (date) => day.truth.find((d) => d.date === date)?.value; | |
| const actual = actualOf(cs.focus_date); | |
| const components = Object.fromEntries(cs.agents.map((a) => [a.system, a.components])); | |
| const live = day.forecasts.filter((f) => f.mode === "live").sort((a, b) => bySystem(a.system, b.system)); | |
| const replay = day.forecasts.filter((f) => f.mode === "mock").sort((a, b) => bySystem(a.system, b.system)); | |
| const strip = h("div", { class: "chart" }); | |
| const rows = live.map((f) => { | |
| const v = value(f, cs.focus_date); | |
| return { system: f.system, point: v.point, q10: v.quantiles?.["0.1"], q90: v.quantiles?.["0.9"], components: components[f.system] }; | |
| }); | |
| if (actual != null) redraws.push(() => anchorStrip(strip, { rows, actual, unit: t.spec.unit })); | |
| const resolved = [...new Set(live.flatMap((f) => f.values.filter((v) => v.crps != null).map((v) => v.target_date)))].sort(); | |
| const best = Object.fromEntries(resolved.map((d) => [d, d3.min([...live, ...replay], (f) => value(f, d)?.crps)])); | |
| const scoreRow = (f, suffix) => h("tr", {}, | |
| h("td", {}, systemCell(f.system), suffix ? h("span", { class: "cell-sub" }, suffix) : null), | |
| resolved.map((d) => { | |
| const c = value(f, d)?.crps; | |
| return h("td", { class: `num${c != null && c === best[d] ? " best" : ""}` }, fmt.value(c)); | |
| })); | |
| return h("div", { class: "case-body" }, | |
| h("div", { class: "chips" }, cs.context.map((c) => h("span", { class: "chip", title: c.note }, `${c.label}: ${c.value}`))), | |
| actual != null ? h("div", {}, h("div", { class: "card-title" }, `Where each forecast for ${dayLabel(cs.focus_date, "dow")} landed`), strip) : null, | |
| h("div", { class: "case-agents" }, cs.agents.map((a) => h("div", { class: "card" }, | |
| h("h3", { class: "card-title" }, systemCell(a.system)), | |
| h("div", { class: "chips", style: { marginBottom: "8px" } }, [ | |
| `${a.effort.minutes} min`, `${a.effort.commands} shell commands`, `${a.effort.searches} web searches`, | |
| a.effort.images ? `${a.effort.images} charts viewed` : null, | |
| ].filter(Boolean).map((text) => h("span", { class: "chip" }, text))), | |
| h("p", { style: { fontWeight: 600, marginBottom: "10px" } }, a.headline), | |
| h("ol", { class: "case-steps" }, a.steps.map((step) => h("li", {}, | |
| h("strong", {}, `${STEP_KIND[step.kind] || step.kind}: ${step.title}. `), h("span", { class: "secondary" }, step.text), | |
| step.formula ? h("div", { class: "formula" }, step.formula) : null, | |
| step.facts ? h("div", { class: "chips", style: { marginTop: "4px" } }, step.facts.map((fact) => h("span", { class: "chip" }, fact))) : null)))))), | |
| resolved.length ? h("div", {}, h("div", { class: "card-title" }, `CRPS (${t.spec.unit}) per resolved target date`), | |
| h("div", { class: "table-wrap" }, h("table", {}, | |
| h("thead", {}, h("tr", {}, h("th", {}, "System"), resolved.map((d) => h("th", { class: "num" }, dayLabel(d, "dow"))))), | |
| h("tbody", {}, | |
| h("tr", { class: "group" }, h("td", { colspan: resolved.length + 1 }, "Live")), live.map((f) => scoreRow(f)), | |
| replay.length ? [h("tr", { class: "group" }, h("td", { colspan: resolved.length + 1 }, "Offline replay of the same day")), replay.map((f) => scoreRow(f, "replay"))] : null, | |
| h("tr", {}, h("td", { class: "muted" }, "Actual"), resolved.map((d) => h("td", { class: "num muted" }, fmt.value(actualOf(d))))))))) : null, | |
| h("div", {}, h("div", { class: "card-title" }, "What this day shows"), h("ul", { class: "secondary" }, cs.takeaways.map((text) => h("li", {}, text))))); | |
| } | |
| function casesSection(t) { | |
| if (!t.case_studies.length) return null; | |
| return h("section", { class: "section" }, | |
| sectionHead("cases", "Case studies", "Single run days written up: what the agents found, how they built their forecasts, and how it went."), | |
| h("div", { class: "cases" }, [...t.case_studies].reverse().map((cs) => { | |
| const body = h("div"); | |
| const details = h("details", { class: "card case" }, | |
| h("summary", {}, h("span", { class: "eyebrow" }, `Run day ${dayLabel(cs.run_date, "long")}`), h("h3", {}, cs.title), | |
| h("p", { class: "secondary small" }, cs.summary), h("span", { class: "open" }, "Read the case study ▸")), | |
| body); | |
| details.addEventListener("toggle", async () => { | |
| if (!details.open || body.childElementCount) return; | |
| const day = await load(`tasks/${t.id}/${cs.run_date}.json`); | |
| body.replaceChildren(caseBody(t, cs, day)); | |
| redraws.forEach((draw) => draw()); | |
| }, { once: false }); | |
| return details; | |
| }))); | |
| } | |
| function filesSection(t) { | |
| const link = (href, text, note) => h("div", {}, h("a", { href, target: "_blank", rel: "noopener" }, `${text} ↗`), h("span", { class: "note" }, note)); | |
| return h("section", { class: "section" }, | |
| sectionHead("files", "Task files"), | |
| h("div", { class: "card files" }, | |
| link(t.links.live_task, "Live Task", " · the ALE Task agents run every run day, with its data-fetching script"), | |
| link(t.links.mock_tasks, "Mock Tasks", " · one offline replay Task per run day"), | |
| link(t.links.results, "Run records", " · forecasts, actual values and every agent trajectory"))); | |
| } | |
| export async function taskPage(app, index, id, params) { | |
| const t = await load(`tasks/${id}.json`); | |
| const state = { mode: params.get("mode") === "mock" ? "mock" : "live", allDays: false }; | |
| const results = h("div"); | |
| const runDays = h("div"); | |
| state.rerender = () => { | |
| redraws.length = 0; | |
| results.replaceChildren(resultsSection(t, state, state.rerender)); | |
| runDays.replaceChildren(runDaysSection(t, state)); | |
| staticDraws.forEach((d) => redraws.push(d)); | |
| redraws.forEach((draw) => draw()); | |
| history.replaceState(null, "", `#/t/${id}${state.mode === "mock" ? "?mode=mock" : ""}`); | |
| }; | |
| const target = targetSection(t); | |
| const staticDraws = [...redraws]; | |
| results.replaceChildren(resultsSection(t, state, state.rerender)); | |
| runDays.replaceChildren(runDaysSection(t, state)); | |
| app.replaceChildren(head(index, t), target, results, runDays, inputsSection(t), casesSection(t), filesSection(t)); | |
| } | |