Spaces:
Running
Running
| import type { BenchmarkRecord, Dataset, FacetGroup, FacetValue, Filters, FilterBlocker, FilterScope } from './types' | |
| import { NONE_VALUE } from './types' | |
| const KNOWN_KEYS = new Set(['date', 'host', 'hardware', 'metrics']) | |
| /** Positional dimension names per known kernel op, for shapes like "4096:4096:4096". */ | |
| const SHAPE_SPECS: Record<string, string[]> = { | |
| matmul: ['m', 'k', 'n'], | |
| attention: ['bs', 'seq', 'heads', 'd'], | |
| 'layer-norm': ['bs', 'seq', 'features'], | |
| softmax: ['bs', 'seq', 'features'], | |
| gelu: ['bs', 'seq', 'features'], | |
| conv1d: ['bs', 'ch', 'len', 'kernel'], | |
| } | |
| /** Compact dimension value for labels: 4096 -> 4k, 131072 -> 128k. */ | |
| export function compactDim(value: string): string { | |
| const n = Number(value) | |
| if (!Number.isFinite(n) || n < 1000) return value | |
| if (n % 1024 === 0) return `${n / 1024}k` | |
| if (n % 1000 === 0) return `${n / 1000}k` | |
| return value | |
| } | |
| /** | |
| * Parse a shape tag into named dimensions. Handles "k=v" pairs | |
| * ("bs=8 seq=4096 heads=32 d=128") and the positional colon form | |
| * ("4096:4096:4096", named via the op's spec, falling back to d1..dN). | |
| */ | |
| export function parseShape(shape: string, op: string): Record<string, string> | null { | |
| const dims: Record<string, string> = {} | |
| if (shape.includes('=')) { | |
| for (const part of shape.split(/[\s,]+/)) { | |
| const eq = part.indexOf('=') | |
| if (eq <= 0) return null | |
| dims[part.slice(0, eq)] = part.slice(eq + 1) | |
| } | |
| return Object.keys(dims).length > 0 ? dims : null | |
| } | |
| if (shape.includes(':')) { | |
| const parts = shape.split(':') | |
| const spec = SHAPE_SPECS[op] | |
| const names = spec && spec.length === parts.length ? spec : parts.map((_, i) => `d${i + 1}`) | |
| parts.forEach((p, i) => (dims[names[i]] = p)) | |
| return dims | |
| } | |
| return null | |
| } | |
| /** | |
| * Replace a record's raw `shape` tag with one `shape.<dim>` pseudo-tag per | |
| * dimension, so every dim gets its own filter chips, group-by option and | |
| * table column. No-op when the shape string cannot be parsed. | |
| */ | |
| function expandShapeTag(r: BenchmarkRecord, op: string): void { | |
| const shape = r.tags.shape | |
| if (shape === undefined) return | |
| const dims = parseShape(shape, op) | |
| if (!dims) return | |
| delete r.tags.shape | |
| for (const [dim, value] of Object.entries(dims)) r.tags[`shape.${dim}`] = value | |
| } | |
| /** | |
| * Canonical shape signature of a record (its "workload" identity): the raw | |
| * `shape` tag, or the parsed dimensions re-joined in order | |
| * ("m=1024 k=1024 n=4096"). Powers the combined shape facet. | |
| */ | |
| export function shapeSignature(r: BenchmarkRecord): string | null { | |
| const dims = Object.entries(r.tags).filter(([k]) => k.startsWith('shape.')) | |
| if (dims.length > 0) return dims.map(([k, v]) => `${k.slice('shape.'.length)}=${v}`).join(' ') | |
| return r.tags.shape ?? null | |
| } | |
| /** Parse one JSONL line into a BenchmarkRecord. Returns null for invalid lines. */ | |
| export function parseRecord(line: string): BenchmarkRecord | null { | |
| const trimmed = line.trim() | |
| if (!trimmed) return null | |
| let raw: Record<string, unknown> | |
| try { | |
| raw = JSON.parse(trimmed) | |
| } catch { | |
| return null | |
| } | |
| if (typeof raw.date !== 'string' || typeof raw.host !== 'string' || typeof raw.hardware !== 'string') { | |
| return null | |
| } | |
| const tags: Record<string, string> = {} | |
| const metrics: Record<string, number> = {} | |
| for (const [key, value] of Object.entries(raw)) { | |
| if (KNOWN_KEYS.has(key)) continue | |
| if (typeof value === 'string') tags[key] = value | |
| else if (typeof value === 'number' && Number.isFinite(value)) metrics[key] = value | |
| } | |
| if (raw.metrics && typeof raw.metrics === 'object') { | |
| for (const [key, value] of Object.entries(raw.metrics as Record<string, unknown>)) { | |
| if (typeof value === 'number' && Number.isFinite(value)) metrics[key] = value | |
| } | |
| } | |
| return { date: raw.date, host: raw.host, hardware: raw.hardware, tags, metrics } | |
| } | |
| async function fetchJsonl(url: string): Promise<string[]> { | |
| const res = await fetch(url, { cache: 'no-store' }) | |
| if (!res.ok) throw new Error(`Failed to fetch ${url}: ${res.status}`) | |
| return (await res.text()).split('\n') | |
| } | |
| /** Identity used to dedupe records present in both flat and partitioned files. */ | |
| function recordId(r: BenchmarkRecord): string { | |
| return JSON.stringify([r.date, r.host, r.hardware, r.tags, r.metrics]) | |
| } | |
| /** Load the data manifest (data/index.json) then every listed dataset. */ | |
| export async function loadDatasets(base: string): Promise<Dataset[]> { | |
| // no-store: new files dropped into /data must show up without a hard refresh. | |
| const manifestRes = await fetch(`${base}data/index.json`, { cache: 'no-store' }) | |
| // A missing manifest just means "no data available yet" — show the empty state. | |
| if (manifestRes.status === 404) return [] | |
| if (!manifestRes.ok) throw new Error(`Failed to fetch data manifest: ${manifestRes.status}`) | |
| // Dev servers may SPA-fallback to index.html (200) for missing files. | |
| let manifest: { categories?: Record<string, string[]> } | |
| try { | |
| manifest = await manifestRes.json() | |
| } catch { | |
| return [] | |
| } | |
| const categories = manifest.categories ?? {} | |
| // Group files into logical benchmarks: the partitioned layout spreads one | |
| // benchmark over <category>/<host>/<date>/<name>.jsonl files (the basename | |
| // is the benchmark name), so every path sharing category+name is one dataset. | |
| const groups = new Map<string, { category: string; name: string; paths: string[] }>() | |
| for (const [category, files] of Object.entries(categories)) { | |
| for (const path of files) { | |
| const file = path.split('/').pop() ?? path | |
| const name = file.replace(/\.jsonl$/, '') | |
| const key = `${category}/${name}` | |
| let group = groups.get(key) | |
| if (!group) groups.set(key, (group = { category, name, paths: [] })) | |
| group.paths.push(path) | |
| } | |
| } | |
| const datasets = await Promise.all( | |
| [...groups.values()].map(async ({ category, name, paths }) => { | |
| const lineSets = await Promise.all(paths.map((path) => fetchJsonl(`${base}data/${path}`))) | |
| const seen = new Set<string>() | |
| const records = lineSets | |
| .flat() | |
| .map((line) => { | |
| const r = parseRecord(line) | |
| if (r) expandShapeTag(r, name) | |
| return r | |
| }) | |
| .filter((r): r is BenchmarkRecord => r !== null) | |
| .filter((r) => { | |
| const id = recordId(r) | |
| if (seen.has(id)) return false | |
| seen.add(id) | |
| return true | |
| }) | |
| .sort((a, b) => a.date.localeCompare(b.date)) | |
| return { name, category, records } | |
| }) | |
| ) | |
| return datasets | |
| } | |
| /** | |
| * Does one record pass the filter set? `skip` ignores one constraint group, | |
| * which is how facet counts are computed: a chip's count is the number of | |
| * records matching every OTHER group's current selection. | |
| */ | |
| export function matchesFilters(r: BenchmarkRecord, f: Filters, skip?: FilterScope): boolean { | |
| if (skip !== 'dates') { | |
| if (f.dateFrom && r.date < f.dateFrom) return false | |
| if (f.dateTo && r.date > f.dateTo) return false | |
| } | |
| if (skip !== 'hosts' && f.hosts.length > 0 && !f.hosts.includes(r.host)) return false | |
| if (skip !== 'hardware' && f.hardware.length > 0 && !f.hardware.includes(r.hardware)) return false | |
| for (const [key, values] of Object.entries(f.tags)) { | |
| if (!values || values.length === 0) continue | |
| if (skip === `tag:${key}`) continue | |
| // The virtual 'shape' key filters on the full shape signature (workload); | |
| // records missing the tag match the (none) facet value — never vanish silently. | |
| const value = (key === 'shape' ? shapeSignature(r) : r.tags[key]) ?? NONE_VALUE | |
| if (!values.includes(value)) return false | |
| } | |
| return true | |
| } | |
| export function applyFilters(records: BenchmarkRecord[], f: Filters): BenchmarkRecord[] { | |
| return records.filter((r) => matchesFilters(r, f)) | |
| } | |
| /** Numeric-aware ordering so shape dims sort 512 < 4096, not "4096" < "512". */ | |
| export function valueSort(values: Iterable<string>): string[] { | |
| const unique = [...new Set(values)] | |
| const real = unique.filter((v) => v !== NONE_VALUE) | |
| const numeric = real.length > 0 && real.every((v) => v.trim() !== '' && !isNaN(Number(v))) | |
| real.sort(numeric ? (a, b) => Number(a) - Number(b) : (a, b) => a.localeCompare(b)) | |
| // The "missing" facet always sorts last. | |
| return unique.includes(NONE_VALUE) ? [...real, NONE_VALUE] : real | |
| } | |
| export function uniqueSorted(values: string[]): string[] { | |
| return [...new Set(values)].sort((a, b) => a.localeCompare(b)) | |
| } | |
| const GROUP_LABELS: Record<string, string> = { hosts: 'Host', hardware: 'Hardware' } | |
| /** Display label: commit/rev abbreviated, shape signatures and dims compact. */ | |
| export function facetLabel(key: string, value: string): string { | |
| if ((key === 'commit' || key === 'rev') && value.length > 12) return value.slice(0, 8) | |
| if (key === 'shape' && value.includes('=')) { | |
| return value | |
| .split(/\s+/) | |
| .map((kv) => { | |
| const eq = kv.indexOf('=') | |
| return eq > 0 ? `${kv.slice(0, eq)}=${compactDim(kv.slice(eq + 1))}` : kv | |
| }) | |
| .join(' ') | |
| } | |
| if (key.startsWith('shape.')) return compactDim(value) | |
| return value | |
| } | |
| /** Tag values of one record, with missing tags exposed as NONE_VALUE. */ | |
| function tagFacetValue(r: BenchmarkRecord, key: string): string { | |
| return r.tags[key] ?? NONE_VALUE | |
| } | |
| /** | |
| * Faceted chip groups: for every group (hosts, hardware, each tag key) list | |
| * all dataset values with the count of records matching the OTHER groups' | |
| * current selections. Zero-count values are rendered disabled by the UI. | |
| */ | |
| export function buildFacets(records: BenchmarkRecord[], f: Filters): FacetGroup[] { | |
| const groups: FacetGroup[] = [] | |
| const make = ( | |
| key: string, | |
| label: string, | |
| scope: FilterScope, | |
| valueOf: (r: BenchmarkRecord) => string | |
| ): void => { | |
| const pool = records.filter((r) => matchesFilters(r, f, scope)) | |
| const counts = new Map<string, number>() | |
| for (const r of pool) { | |
| const v = valueOf(r) | |
| counts.set(v, (counts.get(v) ?? 0) + 1) | |
| } | |
| const selected = | |
| key === 'hosts' ? f.hosts : key === 'hardware' ? f.hardware : (f.tags[key.slice('tag:'.length)] ?? []) | |
| const allValues = valueSort(records.map(valueOf)) | |
| const values: FacetValue[] = allValues.map((v) => ({ | |
| value: v, | |
| label: v === NONE_VALUE ? NONE_VALUE : facetLabel(key.slice('tag:'.length), v), | |
| count: counts.get(v) ?? 0, | |
| selected: selected.includes(v), | |
| })) | |
| groups.push({ key, label, values, selectedCount: selected.length, allCount: pool.length }) | |
| } | |
| make('hosts', GROUP_LABELS.hosts, 'hosts', (r) => r.host) | |
| make('hardware', GROUP_LABELS.hardware, 'hardware', (r) => r.hardware) | |
| // The combined 'shape' facet (workload identity) leads; per-dimension | |
| // groups follow as "shape · <dim>" so dims never lose their context. | |
| const hasShape = records.some((r) => shapeSignature(r) !== null) | |
| const tagKeys = valueSort([...records.flatMap((r) => Object.keys(r.tags)), ...(hasShape ? ['shape'] : [])]) | |
| for (const tagKey of tagKeys) { | |
| const valueOf = | |
| tagKey === 'shape' | |
| ? (r: BenchmarkRecord) => shapeSignature(r) ?? NONE_VALUE | |
| : (r: BenchmarkRecord) => tagFacetValue(r, tagKey) | |
| const distinct = new Set(records.map(valueOf)) | |
| // A single unambiguous value (and no missing records) is not filterable. | |
| if (distinct.size < 2) continue | |
| const label = | |
| tagKey === 'shape' | |
| ? 'shape' | |
| : tagKey.startsWith('shape.') | |
| ? `shape · ${tagKey.slice('shape.'.length)}` | |
| : tagKey.replace(/_/g, ' ') | |
| make(`tag:${tagKey}`, label, `tag:${tagKey}`, valueOf) | |
| } | |
| return groups | |
| } | |
| /** | |
| * Swap inverted date bounds (From > To). Keeps inputs valid as the user types. | |
| */ | |
| export function normalizeFilterDates(f: Filters): Filters { | |
| if (f.dateFrom && f.dateTo && f.dateFrom > f.dateTo) { | |
| return { ...f, dateFrom: f.dateTo, dateTo: f.dateFrom } | |
| } | |
| return f | |
| } | |
| /** | |
| * Drop filter values that no longer exist in the dataset (hot reload may | |
| * remove or replace files), so stale selections can never blank the view. | |
| * Returns the same object when nothing needed pruning. | |
| */ | |
| export function pruneFilters(f: Filters, records: BenchmarkRecord[]): Filters { | |
| const hostSet = new Set(records.map((r) => r.host)) | |
| const hwSet = new Set(records.map((r) => r.hardware)) | |
| const hosts = f.hosts.filter((h) => hostSet.has(h)) | |
| const hardware = f.hardware.filter((h) => hwSet.has(h)) | |
| const tags: Record<string, string[]> = {} | |
| let tagsChanged = false | |
| for (const [key, values] of Object.entries(f.tags)) { | |
| const valid = new Set(records.map((r) => tagFacetValue(r, key))) | |
| const kept = (values ?? []).filter((v) => valid.has(v)) | |
| if (kept.length !== (values ?? []).length) tagsChanged = true | |
| tags[key] = kept | |
| } | |
| const changed = hosts.length !== f.hosts.length || hardware.length !== f.hardware.length || tagsChanged | |
| return changed ? normalizeFilterDates({ ...f, hosts, hardware, tags }) : normalizeFilterDates(f) | |
| } | |
| /** Drop group-by dimensions that vanished; always keep at least 'host'. */ | |
| export function pruneGroupBy(groupBy: string[], records: BenchmarkRecord[]): string[] { | |
| const tagKeys = new Set(records.flatMap((r) => Object.keys(r.tags))) | |
| const valid = (k: string) => k === 'host' || k === 'hardware' || tagKeys.has(k) | |
| const kept = groupBy.filter(valid) | |
| return kept.length > 0 ? kept : ['host'] | |
| } | |
| /** | |
| * Explain why a filter set matches zero records: every active constraint | |
| * group is listed with what it restricts ("date range … has no data", | |
| * "hosts: hf-trx50 only"), so the user sees exactly what blocks. Each entry | |
| * doubles as a one-click recovery target (clear that group). | |
| */ | |
| export function analyzeFilters(records: BenchmarkRecord[], f: Filters): FilterBlocker[] { | |
| const active: { scope: FilterScope; label: string }[] = [] | |
| if (f.dateFrom || f.dateTo) active.push({ scope: 'dates', label: 'dates' }) | |
| if (f.hosts.length > 0) active.push({ scope: 'hosts', label: 'hosts' }) | |
| if (f.hardware.length > 0) active.push({ scope: 'hardware', label: 'hardware' }) | |
| for (const [key, values] of Object.entries(f.tags)) { | |
| if (values && values.length > 0) active.push({ scope: `tag:${key}`, label: key.replace(/^shape\./, '').replace(/_/g, ' ') }) | |
| } | |
| const minDate = records[0]?.date ?? '' | |
| const maxDate = records[records.length - 1]?.date ?? '' | |
| const blockers: FilterBlocker[] = [] | |
| for (const { scope, label } of active) { | |
| // Records matching every OTHER group — used to say whether the range | |
| // itself has no data or just combines badly with the other selections. | |
| const pool = records.filter((r) => matchesFilters(r, f, scope)) | |
| let detail: string | |
| if (scope === 'dates') { | |
| const inRange = pool.filter((r) => (!f.dateFrom || r.date >= f.dateFrom) && (!f.dateTo || r.date <= f.dateTo)) | |
| detail = `date range ${f.dateFrom || minDate}..${f.dateTo || maxDate}${inRange.length === 0 ? ' has no data' : ''}` | |
| } else if (scope === 'hosts' || scope === 'hardware') { | |
| const sel = scope === 'hosts' ? f.hosts : f.hardware | |
| detail = `${label}: ${sel.join(', ')}${sel.length === 1 ? ' only' : ''}` | |
| } else { | |
| const values = f.tags[scope.slice('tag:'.length)] ?? [] | |
| detail = `${label}: ${values.join(', ')}${values.length === 1 ? ' only' : ''}` | |
| } | |
| blockers.push({ key: scope, label, detail }) | |
| } | |
| if (blockers.length === 0 && active.length > 0) { | |
| blockers.push({ key: 'all', label: 'all filters', detail: 'the active filters combine to exclude every run' }) | |
| } | |
| return blockers | |
| } | |
| /** All metric keys present in a record set, sorted for stable display. */ | |
| export function metricKeys(records: BenchmarkRecord[]): string[] { | |
| const keys = new Set<string>() | |
| for (const r of records) for (const k of Object.keys(r.metrics)) keys.add(k) | |
| return [...keys].sort() | |
| } | |
| export interface SeriesPoint { | |
| date: string | |
| t: number | |
| y: number | |
| /** Commits of the runs averaged into this point (when tagged). */ | |
| commits?: string[] | |
| } | |
| export interface Series { | |
| name: string | |
| points: SeriesPoint[] | |
| } | |
| /** Latest vs. previous value of one metric within one series. */ | |
| export interface MetricChange { | |
| latest: number | |
| latestDate: string | |
| previous: number | null | |
| previousDate: string | null | |
| /** Signed percent change (latest vs. previous), null when there is no previous. */ | |
| deltaPct: number | null | |
| /** Commits behind the compared runs, when records carry a commit tag. */ | |
| latestCommits?: string[] | |
| previousCommits?: string[] | |
| } | |
| export interface TrendRow { | |
| /** Series name, from the active group-by dimensions. */ | |
| name: string | |
| /** Metric key -> latest vs. previous change. */ | |
| changes: Record<string, MetricChange> | |
| } | |
| /** | |
| * One row per series (per groupBy combo) with the latest and previous value | |
| * of every metric — the change matrix for spotting regressions at a glance. | |
| */ | |
| export function buildTrends(records: BenchmarkRecord[], metrics: string[], groupBy: string[] = ['host']): TrendRow[] { | |
| const rows = new Map<string, TrendRow>() | |
| for (const metric of metrics) { | |
| for (const s of buildSeries(records, metric, groupBy)) { | |
| const last = s.points[s.points.length - 1] | |
| const prev = s.points[s.points.length - 2] ?? null | |
| let row = rows.get(s.name) | |
| if (!row) rows.set(s.name, (row = { name: s.name, changes: {} })) | |
| row.changes[metric] = { | |
| latest: last.y, | |
| latestDate: last.date, | |
| previous: prev?.y ?? null, | |
| previousDate: prev?.date ?? null, | |
| deltaPct: prev && prev.y !== 0 ? ((last.y - prev.y) / prev.y) * 100 : null, | |
| ...(last.commits ? { latestCommits: last.commits } : {}), | |
| ...(prev?.commits ? { previousCommits: prev.commits } : {}), | |
| } | |
| } | |
| } | |
| return [...rows.values()] | |
| } | |
| /** Series display name from the selected group-by dimensions. */ | |
| function seriesName(r: BenchmarkRecord, groupBy: string[]): string { | |
| const parts: string[] = [] | |
| for (const key of groupBy) { | |
| if (key.startsWith('shape.')) { | |
| // Shape dims read best as labeled, compact pairs: "seq=4k". | |
| const value = r.tags[key] | |
| if (value) parts.push(`${key.slice('shape.'.length)}=${compactDim(value)}`) | |
| continue | |
| } | |
| const value = key === 'host' ? r.host : key === 'hardware' ? r.hardware : r.tags[key] | |
| // Long commit hashes read better abbreviated. | |
| const hash = key === 'commit' || key === 'rev' | |
| if (value) parts.push(hash && value.length > 12 ? value.slice(0, 8) : value) | |
| } | |
| return parts.length > 0 ? parts.join(' · ') : r.host | |
| } | |
| /** | |
| * Group records into time series for a given metric, one series per unique | |
| * combination of the groupBy dimensions ('host', 'hardware', or any tag key). | |
| * When several runs share series+date, the average is used. | |
| */ | |
| export function buildSeries(records: BenchmarkRecord[], metric: string, groupBy: string[] = ['host']): Series[] { | |
| const groups = new Map<string, Map<string, { ys: number[]; commits: Set<string> }>>() | |
| for (const r of records) { | |
| const y = r.metrics[metric] | |
| if (y === undefined) continue | |
| const name = seriesName(r, groupBy) | |
| if (!groups.has(name)) groups.set(name, new Map()) | |
| const byDate = groups.get(name)! | |
| if (!byDate.has(r.date)) byDate.set(r.date, { ys: [], commits: new Set() }) | |
| const bucket = byDate.get(r.date)! | |
| bucket.ys.push(y) | |
| const commit = r.tags.commit ?? r.tags.rev | |
| if (commit) bucket.commits.add(commit) | |
| } | |
| return [...groups.entries()] | |
| .map(([name, byDate]) => ({ | |
| name, | |
| points: [...byDate.entries()] | |
| .map(([date, b]) => ({ | |
| date, | |
| t: new Date(`${date}T00:00:00Z`).getTime(), | |
| y: b.ys.reduce((a, c) => a + c, 0) / b.ys.length, | |
| ...(b.commits.size > 0 ? { commits: [...b.commits] } : {}), | |
| })) | |
| .sort((a, b) => a.t - b.t), | |
| })) | |
| .sort((a, b) => a.name.localeCompare(b.name)) | |
| } | |