/** Pure, browser-independent helpers for score comparison and plot geometry. */ export const MODEL_TYPES = Object.freeze({ VLM: {label: "VLM", color: "#3968bd"}, specialist: {label: "Specialist", color: "#16845c"}, classical: {label: "Classical", color: "#b66b20"}, hybrid: {label: "Hybrid", color: "#8261a8"}, unspecified: {label: "Unspecified", color: "#859183"} }); export const SCORE_AXES = []; for (const aggregation of ["macro", "micro"]) { for (const metric of ["rms", "exact-mapping"]) { for (const measure of ["precision", "recall", "f1"]) { SCORE_AXES.push({ key: [aggregation, metric, measure].join("."), label: (aggregation === "macro" ? "Macro " : "Micro ") + (metric === "rms" ? "RMS " : "Exact Mapping ") + (measure === "f1" ? "F1" : measure[0].toUpperCase() + measure.slice(1)) + " (%)" }); } } } export function modelType(submission = {}) { return Object.hasOwn(MODEL_TYPES, submission.model_type) ? submission.model_type : "unspecified"; } export function axisLabel(axis) { return axis === "parameters" ? "Model parameters (billions)" : SCORE_AXES.find(item => item.key === axis)?.label || axis; } export function scoreValue(result, axis, domain = "all") { if (axis === "parameters") { const value = result.submission?.parameter_count_b; return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null; } const [aggregation, metric, measure] = axis.split("."); const stats = domain === "all" ? result : result.by_domain?.[domain]; const value = stats?.[aggregation]?.[metric]?.[measure]; return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 100 ? value : null; } export function selectPlotPoints(records, {x, y, domain = "all", type = "all"}) { let candidates = 0, omitted = 0; const points = []; for (const record of records) { const category = modelType(record.submission); if (type !== "all" && category !== type) continue; if (domain !== "all" && !record.by_domain?.[domain]) continue; candidates++; const xValue = scoreValue(record, x, domain), yValue = scoreValue(record, y, domain); if (xValue === null || yValue === null) { omitted++; continue; } points.push({record, type: category, x: xValue, y: yValue}); } return {points, candidates, omitted}; } export function xScale(points, axis, logarithmic) { if (axis !== "parameters") return {min: 0, max: 100, log: false, ticks: [0, 20, 40, 60, 80, 100]}; const values = points.map(point => point.x); if (!values.length) return logarithmic ? {min: 0.01, max: 100, log: true, ticks: [0.01, 0.1, 1, 10, 100]} : {min: 0, max: 100, log: false, ticks: [0, 20, 40, 60, 80, 100]}; if (!logarithmic) { const maxValue = Math.max(...values); const power = 10 ** Math.floor(Math.log10(maxValue)); const max = Math.ceil(maxValue / power) * power; return {min: 0, max, log: false, ticks: Array.from({length: 6}, (_, i) => max * i / 5)}; } let lo = Math.floor(Math.log10(Math.min(...values))); let hi = Math.ceil(Math.log10(Math.max(...values))); if (lo === hi) {lo--; hi++;} const ticks = []; const step = Math.max(1, Math.ceil((hi - lo) / 8)); for (let exponent = lo; exponent <= hi; exponent += step) ticks.push(10 ** exponent); if (ticks.at(-1) !== 10 ** hi) ticks.push(10 ** hi); return {min: 10 ** lo, max: 10 ** hi, log: true, ticks}; } export function fraction(value, scale) { const transform = scale.log ? Math.log10 : value => value; return (transform(value) - transform(scale.min)) / (transform(scale.max) - transform(scale.min)); } function validateScores(stats, context) { for (const axis of SCORE_AXES) { if (scoreValue(stats, axis.key) === null) throw new Error(context + ": missing or invalid " + axis.label); } } export function validateLocalSummary(summary, board) { if (!summary || typeof summary !== "object" || Array.isArray(summary)) throw new Error("Expected a summary.json object."); if (summary.dataset_fingerprint !== board.dataset_fingerprint) throw new Error("This summary belongs to a different dataset fingerprint."); if (summary.protocol_version !== "tabulusbench-step2-v1" || summary.evaluation_mode !== "physical-table" || summary.input_mode !== "image-only") throw new Error("Use a physical-table, image-only summary from the published evaluator."); if (summary.population?.fragments !== board.tables) throw new Error("The summary must cover all " + board.tables + " tables."); if (typeof summary.submission?.submission_id !== "string" || !summary.submission.submission_id || typeof summary.submission?.method_name !== "string" || !summary.submission.method_name) throw new Error("Missing submission identity or model name."); validateScores(summary, "Overall scores"); for (const [domain, count] of Object.entries(board.domain_tables)) { if (summary.by_domain?.[domain]?.population?.fragments !== count) throw new Error("Incomplete domain population: " + domain); validateScores(summary.by_domain[domain], domain); } const metadata = summary.submission; if (metadata.model_type != null && !Object.hasOwn(MODEL_TYPES, metadata.model_type)) throw new Error("model_type must be VLM, specialist, classical, hybrid or unspecified."); if (metadata.parameter_count_b != null && !(typeof metadata.parameter_count_b === "number" && Number.isFinite(metadata.parameter_count_b) && metadata.parameter_count_b > 0)) throw new Error("parameter_count_b must be a positive number or null."); return {...summary, _local: true}; }