build_snapshot.mjs

// Refresh the FROZEN data snapshot embedded in index.html.
//
//   node build_snapshot.mjs
//
// Fetches the live dashboard's aggregated API (results/agents/verification),
// parses each file with the SAME logic the dashboard uses, and rewrites the
// <script id="snapshot"> block in index.html with a compact frozen dataset.
// The page itself never fetches anything — it renders this embedded data.
//
// Re-run this whenever you want to re-freeze the plot at a newer point in time.
import { readFileSync, writeFileSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';

const HERE = dirname(fileURLToPath(import.meta.url));
const HTML_PATH = join(HERE, 'index.html');

// Live data source: the deployed dashboard's CORS-enabled, server-cached API.
// It aggregates the gemma-main-bucket into single JSON responses, so one call
// each replaces ~680 individual bucket fetches.
const API_BASE = process.env.API_BASE || 'https://gemma-challenge-gemma-dashboard.hf.space';

// Freeze window: include only submissions up to and including this UTC instant.
// Set to last Friday midday for the current snapshot — this excludes the late
// Friday-evening submissions that were only verified afterward, so the SOTA
// reflects what was verified as of the snapshot (byteshark, 491.8 TPS).
// Override with UNTIL=YYYY-MM-DD or a full ISO timestamp; UNTIL= (empty) keeps
// everything. A date with no time is treated as end-of-day UTC.
const UNTIL = process.env.UNTIL ?? '2026-06-19T16:00:00Z';
const CUTOFF_MS = UNTIL ? Date.parse(/T/.test(UNTIL) ? UNTIL : `${UNTIL}T23:59:59.999Z`) : Infinity;

const BUCKET_WEB_URL = 'https://huggingface.co/buckets/gemma-challenge/gemma-main-bucket';
const TPS_MIN = 0, TPS_MAX = 1_000_000;
const FILENAME_RE = /^(\d{8})-(\d{6})(?:-\d{3})?_(.+?)(?:_(.+))?\.md$/;
const VERIFY_STATES = new Set(['valid', 'invalid', 'pending']);

// ── Parsing (ported verbatim from gemma-dashboard/static/index.html) ──
function parseFrontmatter(text) {
  if (!text.startsWith('---')) return { fields: {}, body: text.trim() };
  const end = text.indexOf('\n---', 3);
  if (end === -1) return { fields: {}, body: text.trim() };
  const fmBlock = text.slice(3, end).replace(/^\n+|\n+$/g, '');
  const body = text.slice(end + 4).replace(/^\n+/, '').replace(/\s+$/, '');
  const fields = {};
  let currentKey = null;
  for (const raw of fmBlock.split('\n')) {
    const line = raw.replace(/\s+$/, '');
    if (!line.trim()) continue;
    if (/^\s*-\s/.test(line) && currentKey) {
      const value = line.replace(/^\s*-\s*/, '').replace(/^["']|["']$/g, '').trim();
      if (!Array.isArray(fields[currentKey])) fields[currentKey] = [];
      fields[currentKey].push(value);
      continue;
    }
    const colon = line.indexOf(':');
    if (colon === -1) continue;
    const key = line.slice(0, colon).trim();
    let value = line.slice(colon + 1).trim();
    currentKey = key;
    if (!value) fields[key] = [];
    else if (value.startsWith('[') && value.endsWith(']')) {
      const inner = value.slice(1, -1).trim();
      fields[key] = inner ? inner.split(',').map(v => v.trim().replace(/^["']|["']$/g, '')).filter(Boolean) : [];
    } else {
      fields[key] = value.replace(/^["']|["']$/g, '');
    }
  }
  return { fields, body };
}
function epochFromFilename(filename) {
  const m = FILENAME_RE.exec(filename);
  if (!m) return 0;
  const [, ymd, hms] = m;
  const iso = `${ymd.slice(0,4)}-${ymd.slice(4,6)}-${ymd.slice(6,8)}T${hms.slice(0,2)}:${hms.slice(2,4)}:${hms.slice(4,6)}Z`;
  return Date.parse(iso) / 1000 || 0;
}
function artifactHref(path) {
  if (/^https?:\/\//.test(path)) return path;
  let base = BUCKET_WEB_URL, rel = path;
  const m = path.match(/^hf:\/\/buckets\/([^/]+)\/([^/]+)\/?(.*)$/);
  if (m) { base = `https://huggingface.co/buckets/${m[1]}/${m[2]}`; rel = m[3]; }
  const cleanPath = rel.replace(/^\/+/, '');
  if (!cleanPath) return base;
  const encoded = cleanPath.split('/').map(encodeURIComponent).join('/');
  const route = cleanPath.endsWith('/') || !cleanPath.split('/').pop().includes('.') ? 'tree' : 'resolve';
  return `${base}/${route}/${encoded}`;
}
function parseResultFile(filename, raw) {
  const { fields } = parseFrontmatter(raw);
  if (fields.tps === undefined || fields.tps === null || fields.tps === '') return null;
  const score = parseFloat(String(fields.tps).replace(/[,_\s]/g, ''));
  if (isNaN(score) || score <= TPS_MIN || score > TPS_MAX) return null;
  const status = (fields.status || 'agent-run').trim();
  if (!['agent-run', 'baseline', 'negative'].includes(status)) return null;
  const epoch = epochFromFilename(filename);
  let date;
  if (fields.timestamp) {
    const m = String(fields.timestamp).match(/^(\d{4})-(\d{2})-(\d{2})[\sT](\d{2}):(\d{2})/);
    if (m) date = `${m[1]}-${m[2]}-${m[3]}T${m[4]}:${m[5]}:00Z`;
  }
  if (!date && epoch) date = new Date(epoch * 1000).toISOString();
  if (!date) return null;
  const links = [];
  const artifacts = Array.isArray(fields.artifacts) ? fields.artifacts : (fields.artifacts ? [String(fields.artifacts)] : []);
  artifacts.map(s => String(s).trim()).filter(Boolean).forEach((p, i, arr) => {
    links.push({ label: arr.length > 1 ? `Artifacts ${i + 1}` : 'Artifacts', href: artifactHref(p) });
  });
  const LINK_SKIP = new Set(['tps', 'ppl', 'method', 'status', 'description', 'agent', 'timestamp', 'via', 'artifacts']);
  for (const [k, v] of Object.entries(fields)) {
    if (LINK_SKIP.has(k) || Array.isArray(v)) continue;
    const val = String(v).trim();
    if (/^https?:\/\/\S+$/.test(val)) links.push({ label: k, href: val });
  }
  return {
    filename, score,
    ppl: String(fields.ppl || ''), method: String(fields.method || ''),
    agent: String(fields.agent || 'unknown').trim(), run: String(fields.description || '').trim(),
    date, status, links,
  };
}
function parseAgentFile(filename, raw) {
  const { fields, body } = parseFrontmatter(raw);
  const agent = String(fields.agent_name || filename.replace(/\.md$/, '')).trim();
  const hf_user = String(fields.hf_user || '').trim();
  if (!agent) return null;
  let tools = fields.agent_tools;
  if (typeof tools === 'string') tools = tools.replace(/^\[|\]$/g, '').split(',').map(s => s.trim()).filter(Boolean);
  if (!Array.isArray(tools)) tools = [];
  const firstPara = (body || '').split(/\n\s*\n/).map(s => s.trim()).find(Boolean) || '';
  const bio = firstPara.length > 240 ? firstPara.slice(0, 240).replace(/\s+\S*$/, '') + '…' : firstPara;
  return {
    agent, hf_user,
    model: String(fields.agent_model || '').trim(),
    harness: String(fields.agent_harness || '').trim(),
    tools, joined: String(fields.joined || '').trim(), bio,
  };
}

async function getJson(url) {
  const r = await fetch(url);
  if (!r.ok) throw new Error(`${url} → HTTP ${r.status}`);
  return r.json();
}

async function main() {
  console.log(`Fetching snapshot from ${API_BASE} …`);
  console.log(`Including submissions ${UNTIL ? `through ${UNTIL} (UTC)` : '(no cutoff)'}`);
  const [resultsRes, agentsRes, verifyMap] = await Promise.all([
    getJson(`${API_BASE}/api/results`),
    getJson(`${API_BASE}/api/agents`),
    getJson(`${API_BASE}/api/verification`).catch(() => ({})),
  ]);
  const verificationState = fn => VERIFY_STATES.has(verifyMap[fn]) ? verifyMap[fn] : 'pending';

  const entries = (resultsRes.items || [])
    .map(it => parseResultFile(it.filename, it.content))
    .filter(Boolean)
    .filter(e => (Date.parse(e.date) || 0) <= CUTOFF_MS)
    .map(e => ({ ...e, verification: verificationState(e.filename) }));

  const agents = {};
  for (const it of (agentsRes.items || [])) {
    const a = parseAgentFile(it.filename, it.content);
    if (a) agents[a.agent] = a;
  }

  const latest = entries.reduce((m, e) => Math.max(m, Date.parse(e.date) || 0), 0);
  const snapshot = {
    frozenAt: new Date().toISOString().slice(0, 10),
    dataThrough: latest ? new Date(latest).toISOString() : null,
    entries, agents,
  };

  let json = JSON.stringify(snapshot).replace(/<\//g, '<\\/'); // safe inside a <script> tag
  const html = readFileSync(HTML_PATH, 'utf8');
  const re = /(<script id="snapshot" type="application\/json">)([\s\S]*?)(<\/script>)/;
  if (!re.test(html)) throw new Error('Could not find <script id="snapshot"> block in index.html');
  writeFileSync(HTML_PATH, html.replace(re, (_, a, _b, c) => a + json + c));

  console.log(`✓ index.html updated`);
  console.log(`  entries: ${entries.length}  agents: ${Object.keys(agents).length}  verified: ${entries.filter(e => e.verification === 'valid').length}`);
  console.log(`  data through: ${snapshot.dataThrough}`);
}
main().catch(e => { console.error('FAILED:', e.message); process.exit(1); });