import assert from "node:assert/strict"; import { access, readFile } from "node:fs/promises"; import { resolve } from "node:path"; const root = resolve(import.meta.dirname, ".."); const splitFiles = { train: { path: "data/train.jsonl", count: 840 }, validation: { path: "data/validation.jsonl", count: 120 }, test: { path: "data/test.jsonl", count: 240 }, }; const requiredFields = [ "id", "pair_id", "scenario_id", "text", "label", "category", "attack_family", "pair_family", "source_context", "risk_domain", "target_boundary", "expected_action", "source_type", "language", "split", ]; const expectedUiIds = [ "tab-pair", "pair-scenario", "pair-family", "benign-text", "attack-text", "tab-scenarios", "scenario-list", "scenario-detail", "tab-records", "record-filter-form", "record-results-body", "record-detail", "tab-method", "citation-text", "app-status", ]; async function readJsonl(relativePath) { const raw = await readFile(resolve(root, relativePath), "utf8"); return raw .split(/\r?\n/) .filter(Boolean) .map((line, index) => { try { return JSON.parse(line); } catch (error) { throw new Error(`${relativePath}:${index + 1} is not valid JSON: ${error.message}`); } }); } function groupBy(items, key) { const groups = new Map(); for (const item of items) { const value = item[key]; if (!groups.has(value)) groups.set(value, []); groups.get(value).push(item); } return groups; } for (const file of [ "index.html", "assets/styles.css", "assets/app.js", "assets/altaysec-mark.svg", "source/scenarios.jsonl", ...Object.values(splitFiles).map(({ path }) => path), ]) { await access(resolve(root, file)); } const rows = []; for (const [split, definition] of Object.entries(splitFiles)) { const splitRows = await readJsonl(definition.path); assert.equal(splitRows.length, definition.count, `${split} row count`); assert(splitRows.every((row) => row.split === split), `${split} contains a mismatched split value`); rows.push(...splitRows); } const scenarios = await readJsonl("source/scenarios.jsonl"); assert.equal(rows.length, 1200, "total row count"); assert.equal(scenarios.length, 50, "scenario count"); const ids = new Set(); for (const [index, row] of rows.entries()) { for (const field of requiredFields) { assert(Object.hasOwn(row, field), `row ${index + 1} is missing ${field}`); } assert(!ids.has(row.id), `duplicate row id: ${row.id}`); ids.add(row.id); assert([0, 1].includes(row.label), `${row.id} has an invalid label`); assert.equal(row.language, "en", `${row.id} is not English`); assert.equal(row.source_type, "synthetic_curated", `${row.id} has an unexpected source type`); assert(row.text.trim().length > 0, `${row.id} has empty text`); } const labelGroups = groupBy(rows, "label"); assert.equal(labelGroups.get(0)?.length, 600, "benign row count"); assert.equal(labelGroups.get(1)?.length, 600, "attack row count"); const pairs = groupBy(rows, "pair_id"); assert.equal(pairs.size, 600, "pair count"); for (const [pairId, pairRows] of pairs) { assert.equal(pairRows.length, 2, `${pairId} must contain two rows`); assert.deepEqual( [...pairRows.map((row) => row.label)].sort(), [0, 1], `${pairId} must contain one benign and one attack row`, ); for (const field of [ "scenario_id", "split", "pair_family", "source_context", "risk_domain", "target_boundary", ]) { assert.equal(new Set(pairRows.map((row) => row[field])).size, 1, `${pairId} mismatches ${field}`); } const attack = pairRows.find((row) => row.label === 1); assert.equal(attack.attack_family, attack.pair_family, `${pairId} attack family mismatch`); } const scenarioIds = new Set(scenarios.map((scenario) => scenario.scenario_id)); assert.equal(scenarioIds.size, 50, "scenario identifiers must be unique"); const rowsByScenario = groupBy(rows, "scenario_id"); assert.equal(rowsByScenario.size, 50, "dataset scenario count"); for (const [scenarioId, scenarioRows] of rowsByScenario) { assert(scenarioIds.has(scenarioId), `${scenarioId} is missing from scenarios.jsonl`); assert.equal(scenarioRows.length, 24, `${scenarioId} row count`); assert.equal(new Set(scenarioRows.map((row) => row.pair_id)).size, 12, `${scenarioId} pair count`); assert.equal(new Set(scenarioRows.map((row) => row.pair_family)).size, 12, `${scenarioId} family count`); assert.equal(new Set(scenarioRows.map((row) => row.split)).size, 1, `${scenarioId} crosses splits`); } const attackFamilies = groupBy(rows.filter((row) => row.label === 1), "attack_family"); assert.equal(attackFamilies.size, 12, "attack family count"); for (const [family, familyRows] of attackFamilies) { assert.equal(familyRows.length, 50, `${family} attack count`); } const html = await readFile(resolve(root, "index.html"), "utf8"); for (const id of expectedUiIds) { assert(html.includes(`id="${id}"`), `index.html is missing #${id}`); } assert(html.includes('src="assets/app.js"'), "index.html does not load app.js"); assert(html.includes('href="assets/styles.css"'), "index.html does not load styles.css"); console.log("Agentic Explorer validation passed"); console.log(" 1,200 rows | 600 pairs | 50 scenarios | 12 attack families"); console.log(" train 840 | validation 120 | test 240 | labels 600/600");