File size: 5,621 Bytes
2d64baa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
/** Pure, browser-independent helpers for score comparison and plot geometry. */
export const MODEL_TYPES = Object.freeze({
  VLM: {label: "VLM", color: "#3968bd"},
  specialist: {label: "Specialist", color: "#16845c"},
  classical: {label: "Classical", color: "#b66b20"},
  hybrid: {label: "Hybrid", color: "#8261a8"},
  unspecified: {label: "Unspecified", color: "#859183"}
});
export const SCORE_AXES = [];
for (const aggregation of ["macro", "micro"]) {
  for (const metric of ["rms", "exact-mapping"]) {
    for (const measure of ["precision", "recall", "f1"]) {
      SCORE_AXES.push({
        key: [aggregation, metric, measure].join("."),
        label: (aggregation === "macro" ? "Macro " : "Micro ") +
          (metric === "rms" ? "RMS " : "Exact Mapping ") +
          (measure === "f1" ? "F1" : measure[0].toUpperCase() + measure.slice(1)) + " (%)"
      });
    }
  }
}
export function modelType(submission = {}) {
  return Object.hasOwn(MODEL_TYPES, submission.model_type) ? submission.model_type : "unspecified";
}
export function axisLabel(axis) {
  return axis === "parameters" ? "Model parameters (billions)" : SCORE_AXES.find(item => item.key === axis)?.label || axis;
}
export function scoreValue(result, axis, domain = "all") {
  if (axis === "parameters") {
    const value = result.submission?.parameter_count_b;
    return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null;
  }
  const [aggregation, metric, measure] = axis.split(".");
  const stats = domain === "all" ? result : result.by_domain?.[domain];
  const value = stats?.[aggregation]?.[metric]?.[measure];
  return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 100 ? value : null;
}
export function selectPlotPoints(records, {x, y, domain = "all", type = "all"}) {
  let candidates = 0, omitted = 0;
  const points = [];
  for (const record of records) {
    const category = modelType(record.submission);
    if (type !== "all" && category !== type) continue;
    if (domain !== "all" && !record.by_domain?.[domain]) continue;
    candidates++;
    const xValue = scoreValue(record, x, domain), yValue = scoreValue(record, y, domain);
    if (xValue === null || yValue === null) { omitted++; continue; }
    points.push({record, type: category, x: xValue, y: yValue});
  }
  return {points, candidates, omitted};
}
export function xScale(points, axis, logarithmic) {
  if (axis !== "parameters") return {min: 0, max: 100, log: false, ticks: [0, 20, 40, 60, 80, 100]};
  const values = points.map(point => point.x);
  if (!values.length) return logarithmic
    ? {min: 0.01, max: 100, log: true, ticks: [0.01, 0.1, 1, 10, 100]}
    : {min: 0, max: 100, log: false, ticks: [0, 20, 40, 60, 80, 100]};
  if (!logarithmic) {
    const maxValue = Math.max(...values);
    const power = 10 ** Math.floor(Math.log10(maxValue));
    const max = Math.ceil(maxValue / power) * power;
    return {min: 0, max, log: false, ticks: Array.from({length: 6}, (_, i) => max * i / 5)};
  }
  let lo = Math.floor(Math.log10(Math.min(...values)));
  let hi = Math.ceil(Math.log10(Math.max(...values)));
  if (lo === hi) {lo--; hi++;}
  const ticks = [];
  const step = Math.max(1, Math.ceil((hi - lo) / 8));
  for (let exponent = lo; exponent <= hi; exponent += step) ticks.push(10 ** exponent);
  if (ticks.at(-1) !== 10 ** hi) ticks.push(10 ** hi);
  return {min: 10 ** lo, max: 10 ** hi, log: true, ticks};
}
export function fraction(value, scale) {
  const transform = scale.log ? Math.log10 : value => value;
  return (transform(value) - transform(scale.min)) / (transform(scale.max) - transform(scale.min));
}
function validateScores(stats, context) {
  for (const axis of SCORE_AXES) {
    if (scoreValue(stats, axis.key) === null) throw new Error(context + ": missing or invalid " + axis.label);
  }
}
export function validateLocalSummary(summary, board) {
  if (!summary || typeof summary !== "object" || Array.isArray(summary)) throw new Error("Expected a summary.json object.");
  if (summary.dataset_fingerprint !== board.dataset_fingerprint) throw new Error("This summary belongs to a different dataset fingerprint.");
  if (summary.protocol_version !== "tabulusbench-step2-v1" ||
      summary.evaluation_mode !== "physical-table" || summary.input_mode !== "image-only")
    throw new Error("Use a physical-table, image-only summary from the published evaluator.");
  if (summary.population?.fragments !== board.tables) throw new Error("The summary must cover all " + board.tables + " tables.");
  if (typeof summary.submission?.submission_id !== "string" || !summary.submission.submission_id ||
      typeof summary.submission?.method_name !== "string" || !summary.submission.method_name)
    throw new Error("Missing submission identity or model name.");
  validateScores(summary, "Overall scores");
  for (const [domain, count] of Object.entries(board.domain_tables)) {
    if (summary.by_domain?.[domain]?.population?.fragments !== count) throw new Error("Incomplete domain population: " + domain);
    validateScores(summary.by_domain[domain], domain);
  }
  const metadata = summary.submission;
  if (metadata.model_type != null && !Object.hasOwn(MODEL_TYPES, metadata.model_type))
    throw new Error("model_type must be VLM, specialist, classical, hybrid or unspecified.");
  if (metadata.parameter_count_b != null &&
      !(typeof metadata.parameter_count_b === "number" && Number.isFinite(metadata.parameter_count_b) && metadata.parameter_count_b > 0))
    throw new Error("parameter_count_b must be a positive number or null.");
  return {...summary, _local: true};
}