scripts/validate-model-data.mjs

import fs from "node:fs";
import path from "node:path";
import vm from "node:vm";

const root = path.resolve(import.meta.dirname, "..");
const dataPath = path.join(root, "assets", "local-frontier-model-data.js");
const source = fs.readFileSync(dataPath, "utf8");
const context = { window: {} };
vm.runInNewContext(source, context, { filename: dataPath });

const data = context.window.LOCAL_FRONTIER_MODEL_DATA;
const errors = [];
const HF_REPO_ID_RE =
  /^[A-Za-z0-9][A-Za-z0-9._-]*\/[A-Za-z0-9][A-Za-z0-9._-]*$/;
const LEGACY_ALIAS_RE = /^[a-z0-9][a-z0-9-]*$/;

function fail(path, message) {
  errors.push(`${path}: ${message}`);
}

function isPositiveNumber(value) {
  return Number.isFinite(Number(value)) && Number(value) > 0;
}

function isNonNegativeNumber(value) {
  return Number.isFinite(Number(value)) && Number(value) >= 0;
}

if (!data || typeof data !== "object") {
  fail("LOCAL_FRONTIER_MODEL_DATA", "missing data object");
} else {
  if (!Array.isArray(data.models) || data.models.length === 0) {
    fail("models", "must contain at least one model");
  }
  const ids = new Set();
  const aliases = new Set();
  for (const [index, model] of (data.models || []).entries()) {
    const pathBase = `models[${index}]`;
    if (!model.id || !HF_REPO_ID_RE.test(model.id))
      fail(
        `${pathBase}.id`,
        "must be a Hugging Face repo id like namespace/model",
      );
    if (ids.has(model.id)) fail(`${pathBase}.id`, "duplicate id");
    ids.add(model.id);
    for (const [aliasIndex, alias] of (model.aliases || []).entries()) {
      if (!LEGACY_ALIAS_RE.test(alias))
        fail(`${pathBase}.aliases[${aliasIndex}]`, "invalid legacy alias");
      if (ids.has(alias))
        fail(
          `${pathBase}.aliases[${aliasIndex}]`,
          "alias collides with canonical id",
        );
      if (aliases.has(alias))
        fail(`${pathBase}.aliases[${aliasIndex}]`, "duplicate alias");
      aliases.add(alias);
    }
    for (const key of ["name", "short_name", "license"]) {
      if (!model[key]) fail(`${pathBase}.${key}`, "required");
    }
    if (!model.hf_pipeline_tag) fail(`${pathBase}.hf_pipeline_tag`, "required");
    const minDownloads = data.assumptions?.min_hf_downloads ?? 100000;
    if (
      !Number.isInteger(model.hf_downloads) ||
      model.hf_downloads < minDownloads
    ) {
      fail(`${pathBase}.hf_downloads`, `must be an integer >= ${minDownloads}`);
    }
    const arch = model.architecture || {};
    for (const key of ["type", "detail"]) {
      if (!arch[key]) fail(`${pathBase}.architecture.${key}`, "required");
    }
    for (const key of ["total_params_b", "active_params_b"]) {
      if (!isPositiveNumber(arch[key]))
        fail(`${pathBase}.architecture.${key}`, "must be positive");
    }
    if (
      arch.max_context_tokens !== null &&
      (!Number.isInteger(arch.max_context_tokens) ||
        arch.max_context_tokens <= 0)
    ) {
      fail(
        `${pathBase}.architecture.max_context_tokens`,
        "must be null or a positive integer",
      );
    }
    const adapter = model.adapter || {};
    if (!["dense", "moe", "recurrent", "hybrid"].includes(adapter.kind)) {
      fail(`${pathBase}.adapter.kind`, "unknown adapter kind");
    }
    for (const key of [
      "weight_bytes_per_param",
      "resident_weight_gb",
      "active_weight_gb",
    ]) {
      if (!isPositiveNumber(adapter[key]))
        fail(`${pathBase}.adapter.${key}`, "must be positive");
    }
    for (const key of [
      "kv_alloc_gb_per_1k_tokens",
      "kv_read_gb_per_1k_tokens",
      "kv_alloc_fixed_gb",
      "kv_read_fixed_gb",
    ]) {
      if (adapter[key] !== undefined && !isNonNegativeNumber(adapter[key])) {
        fail(`${pathBase}.adapter.${key}`, "must be non-negative when present");
      }
    }
    if (adapter.kind === "moe") {
      for (const key of ["routed_experts", "routed_experts_per_token"]) {
        if (!Number.isInteger(arch[key]) || arch[key] <= 0)
          fail(`${pathBase}.architecture.${key}`, "required for MoE");
      }
    }
    if (!Array.isArray(model.sources) || model.sources.length === 0) {
      fail(`${pathBase}.sources`, "at least one source is required");
    } else {
      for (const [sourceIndex, sourceRow] of model.sources.entries()) {
        if (!sourceRow.label)
          fail(`${pathBase}.sources[${sourceIndex}].label`, "required");
        try {
          new URL(sourceRow.url);
        } catch {
          fail(`${pathBase}.sources[${sourceIndex}].url`, "invalid URL");
        }
      }
    }
  }
}

if (errors.length) {
  console.error(errors.join("\n"));
  process.exit(1);
}

console.log(`validated ${data.models.length} models`);