scripts/validate-static.mjs

import assert from "node:assert/strict";
import { access, readFile } from "node:fs/promises";
import { resolve } from "node:path";

const root = resolve(import.meta.dirname, "..");

const splitFiles = {
  train: { path: "data/train.jsonl", count: 840 },
  validation: { path: "data/validation.jsonl", count: 120 },
  test: { path: "data/test.jsonl", count: 240 },
};

const requiredFields = [
  "id",
  "pair_id",
  "scenario_id",
  "text",
  "label",
  "category",
  "attack_family",
  "pair_family",
  "source_context",
  "risk_domain",
  "target_boundary",
  "expected_action",
  "source_type",
  "language",
  "split",
];

const expectedUiIds = [
  "tab-pair",
  "pair-scenario",
  "pair-family",
  "benign-text",
  "attack-text",
  "tab-scenarios",
  "scenario-list",
  "scenario-detail",
  "tab-records",
  "record-filter-form",
  "record-results-body",
  "record-detail",
  "tab-method",
  "citation-text",
  "app-status",
];

async function readJsonl(relativePath) {
  const raw = await readFile(resolve(root, relativePath), "utf8");
  return raw
    .split(/\r?\n/)
    .filter(Boolean)
    .map((line, index) => {
      try {
        return JSON.parse(line);
      } catch (error) {
        throw new Error(`${relativePath}:${index + 1} is not valid JSON: ${error.message}`);
      }
    });
}

function groupBy(items, key) {
  const groups = new Map();
  for (const item of items) {
    const value = item[key];
    if (!groups.has(value)) groups.set(value, []);
    groups.get(value).push(item);
  }
  return groups;
}

for (const file of [
  "index.html",
  "assets/styles.css",
  "assets/app.js",
  "assets/altaysec-mark.svg",
  "source/scenarios.jsonl",
  ...Object.values(splitFiles).map(({ path }) => path),
]) {
  await access(resolve(root, file));
}

const rows = [];
for (const [split, definition] of Object.entries(splitFiles)) {
  const splitRows = await readJsonl(definition.path);
  assert.equal(splitRows.length, definition.count, `${split} row count`);
  assert(splitRows.every((row) => row.split === split), `${split} contains a mismatched split value`);
  rows.push(...splitRows);
}

const scenarios = await readJsonl("source/scenarios.jsonl");
assert.equal(rows.length, 1200, "total row count");
assert.equal(scenarios.length, 50, "scenario count");

const ids = new Set();
for (const [index, row] of rows.entries()) {
  for (const field of requiredFields) {
    assert(Object.hasOwn(row, field), `row ${index + 1} is missing ${field}`);
  }
  assert(!ids.has(row.id), `duplicate row id: ${row.id}`);
  ids.add(row.id);
  assert([0, 1].includes(row.label), `${row.id} has an invalid label`);
  assert.equal(row.language, "en", `${row.id} is not English`);
  assert.equal(row.source_type, "synthetic_curated", `${row.id} has an unexpected source type`);
  assert(row.text.trim().length > 0, `${row.id} has empty text`);
}

const labelGroups = groupBy(rows, "label");
assert.equal(labelGroups.get(0)?.length, 600, "benign row count");
assert.equal(labelGroups.get(1)?.length, 600, "attack row count");

const pairs = groupBy(rows, "pair_id");
assert.equal(pairs.size, 600, "pair count");
for (const [pairId, pairRows] of pairs) {
  assert.equal(pairRows.length, 2, `${pairId} must contain two rows`);
  assert.deepEqual(
    [...pairRows.map((row) => row.label)].sort(),
    [0, 1],
    `${pairId} must contain one benign and one attack row`,
  );

  for (const field of [
    "scenario_id",
    "split",
    "pair_family",
    "source_context",
    "risk_domain",
    "target_boundary",
  ]) {
    assert.equal(new Set(pairRows.map((row) => row[field])).size, 1, `${pairId} mismatches ${field}`);
  }

  const attack = pairRows.find((row) => row.label === 1);
  assert.equal(attack.attack_family, attack.pair_family, `${pairId} attack family mismatch`);
}

const scenarioIds = new Set(scenarios.map((scenario) => scenario.scenario_id));
assert.equal(scenarioIds.size, 50, "scenario identifiers must be unique");

const rowsByScenario = groupBy(rows, "scenario_id");
assert.equal(rowsByScenario.size, 50, "dataset scenario count");
for (const [scenarioId, scenarioRows] of rowsByScenario) {
  assert(scenarioIds.has(scenarioId), `${scenarioId} is missing from scenarios.jsonl`);
  assert.equal(scenarioRows.length, 24, `${scenarioId} row count`);
  assert.equal(new Set(scenarioRows.map((row) => row.pair_id)).size, 12, `${scenarioId} pair count`);
  assert.equal(new Set(scenarioRows.map((row) => row.pair_family)).size, 12, `${scenarioId} family count`);
  assert.equal(new Set(scenarioRows.map((row) => row.split)).size, 1, `${scenarioId} crosses splits`);
}

const attackFamilies = groupBy(rows.filter((row) => row.label === 1), "attack_family");
assert.equal(attackFamilies.size, 12, "attack family count");
for (const [family, familyRows] of attackFamilies) {
  assert.equal(familyRows.length, 50, `${family} attack count`);
}

const html = await readFile(resolve(root, "index.html"), "utf8");
for (const id of expectedUiIds) {
  assert(html.includes(`id="${id}"`), `index.html is missing #${id}`);
}
assert(html.includes('src="assets/app.js"'), "index.html does not load app.js");
assert(html.includes('href="assets/styles.css"'), "index.html does not load styles.css");

console.log("Agentic Explorer validation passed");
console.log("  1,200 rows | 600 pairs | 50 scenarios | 12 attack families");
console.log("  train 840 | validation 120 | test 240 | labels 600/600");