jdsouza's picture
Add model-type colors and interactive reconstruction comparison plotter
2d64baa verified
Raw History Blame Contribute Delete
5.62 kB
/** Pure, browser-independent helpers for score comparison and plot geometry. */
export const MODEL_TYPES = Object.freeze({
VLM: {label: "VLM", color: "#3968bd"},
specialist: {label: "Specialist", color: "#16845c"},
classical: {label: "Classical", color: "#b66b20"},
hybrid: {label: "Hybrid", color: "#8261a8"},
unspecified: {label: "Unspecified", color: "#859183"}
});
export const SCORE_AXES = [];
for (const aggregation of ["macro", "micro"]) {
for (const metric of ["rms", "exact-mapping"]) {
for (const measure of ["precision", "recall", "f1"]) {
SCORE_AXES.push({
key: [aggregation, metric, measure].join("."),
label: (aggregation === "macro" ? "Macro " : "Micro ") +
(metric === "rms" ? "RMS " : "Exact Mapping ") +
(measure === "f1" ? "F1" : measure[0].toUpperCase() + measure.slice(1)) + " (%)"
});
}
}
}
export function modelType(submission = {}) {
return Object.hasOwn(MODEL_TYPES, submission.model_type) ? submission.model_type : "unspecified";
}
export function axisLabel(axis) {
return axis === "parameters" ? "Model parameters (billions)" : SCORE_AXES.find(item => item.key === axis)?.label || axis;
}
export function scoreValue(result, axis, domain = "all") {
if (axis === "parameters") {
const value = result.submission?.parameter_count_b;
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null;
}
const [aggregation, metric, measure] = axis.split(".");
const stats = domain === "all" ? result : result.by_domain?.[domain];
const value = stats?.[aggregation]?.[metric]?.[measure];
return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 100 ? value : null;
}
export function selectPlotPoints(records, {x, y, domain = "all", type = "all"}) {
let candidates = 0, omitted = 0;
const points = [];
for (const record of records) {
const category = modelType(record.submission);
if (type !== "all" && category !== type) continue;
if (domain !== "all" && !record.by_domain?.[domain]) continue;
candidates++;
const xValue = scoreValue(record, x, domain), yValue = scoreValue(record, y, domain);
if (xValue === null || yValue === null) { omitted++; continue; }
points.push({record, type: category, x: xValue, y: yValue});
}
return {points, candidates, omitted};
}
export function xScale(points, axis, logarithmic) {
if (axis !== "parameters") return {min: 0, max: 100, log: false, ticks: [0, 20, 40, 60, 80, 100]};
const values = points.map(point => point.x);
if (!values.length) return logarithmic
? {min: 0.01, max: 100, log: true, ticks: [0.01, 0.1, 1, 10, 100]}
: {min: 0, max: 100, log: false, ticks: [0, 20, 40, 60, 80, 100]};
if (!logarithmic) {
const maxValue = Math.max(...values);
const power = 10 ** Math.floor(Math.log10(maxValue));
const max = Math.ceil(maxValue / power) * power;
return {min: 0, max, log: false, ticks: Array.from({length: 6}, (_, i) => max * i / 5)};
}
let lo = Math.floor(Math.log10(Math.min(...values)));
let hi = Math.ceil(Math.log10(Math.max(...values)));
if (lo === hi) {lo--; hi++;}
const ticks = [];
const step = Math.max(1, Math.ceil((hi - lo) / 8));
for (let exponent = lo; exponent <= hi; exponent += step) ticks.push(10 ** exponent);
if (ticks.at(-1) !== 10 ** hi) ticks.push(10 ** hi);
return {min: 10 ** lo, max: 10 ** hi, log: true, ticks};
}
export function fraction(value, scale) {
const transform = scale.log ? Math.log10 : value => value;
return (transform(value) - transform(scale.min)) / (transform(scale.max) - transform(scale.min));
}
function validateScores(stats, context) {
for (const axis of SCORE_AXES) {
if (scoreValue(stats, axis.key) === null) throw new Error(context + ": missing or invalid " + axis.label);
}
}
export function validateLocalSummary(summary, board) {
if (!summary || typeof summary !== "object" || Array.isArray(summary)) throw new Error("Expected a summary.json object.");
if (summary.dataset_fingerprint !== board.dataset_fingerprint) throw new Error("This summary belongs to a different dataset fingerprint.");
if (summary.protocol_version !== "tabulusbench-step2-v1" ||
summary.evaluation_mode !== "physical-table" || summary.input_mode !== "image-only")
throw new Error("Use a physical-table, image-only summary from the published evaluator.");
if (summary.population?.fragments !== board.tables) throw new Error("The summary must cover all " + board.tables + " tables.");
if (typeof summary.submission?.submission_id !== "string" || !summary.submission.submission_id ||
typeof summary.submission?.method_name !== "string" || !summary.submission.method_name)
throw new Error("Missing submission identity or model name.");
validateScores(summary, "Overall scores");
for (const [domain, count] of Object.entries(board.domain_tables)) {
if (summary.by_domain?.[domain]?.population?.fragments !== count) throw new Error("Incomplete domain population: " + domain);
validateScores(summary.by_domain[domain], domain);
}
const metadata = summary.submission;
if (metadata.model_type != null && !Object.hasOwn(MODEL_TYPES, metadata.model_type))
throw new Error("model_type must be VLM, specialist, classical, hybrid or unspecified.");
if (metadata.parameter_count_b != null &&
!(typeof metadata.parameter_count_b === "number" && Number.isFinite(metadata.parameter_count_b) && metadata.parameter_count_b > 0))
throw new Error("parameter_count_b must be a positive number or null.");
return {...summary, _local: true};
}