File size: 5,407 Bytes
8673c10 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 | import assert from "node:assert/strict";
import { access, readFile } from "node:fs/promises";
import { resolve } from "node:path";
const root = resolve(import.meta.dirname, "..");
const splitFiles = {
train: { path: "data/train.jsonl", count: 840 },
validation: { path: "data/validation.jsonl", count: 120 },
test: { path: "data/test.jsonl", count: 240 },
};
const requiredFields = [
"id",
"pair_id",
"scenario_id",
"text",
"label",
"category",
"attack_family",
"pair_family",
"source_context",
"risk_domain",
"target_boundary",
"expected_action",
"source_type",
"language",
"split",
];
const expectedUiIds = [
"tab-pair",
"pair-scenario",
"pair-family",
"benign-text",
"attack-text",
"tab-scenarios",
"scenario-list",
"scenario-detail",
"tab-records",
"record-filter-form",
"record-results-body",
"record-detail",
"tab-method",
"citation-text",
"app-status",
];
async function readJsonl(relativePath) {
const raw = await readFile(resolve(root, relativePath), "utf8");
return raw
.split(/\r?\n/)
.filter(Boolean)
.map((line, index) => {
try {
return JSON.parse(line);
} catch (error) {
throw new Error(`${relativePath}:${index + 1} is not valid JSON: ${error.message}`);
}
});
}
function groupBy(items, key) {
const groups = new Map();
for (const item of items) {
const value = item[key];
if (!groups.has(value)) groups.set(value, []);
groups.get(value).push(item);
}
return groups;
}
for (const file of [
"index.html",
"assets/styles.css",
"assets/app.js",
"assets/altaysec-mark.svg",
"source/scenarios.jsonl",
...Object.values(splitFiles).map(({ path }) => path),
]) {
await access(resolve(root, file));
}
const rows = [];
for (const [split, definition] of Object.entries(splitFiles)) {
const splitRows = await readJsonl(definition.path);
assert.equal(splitRows.length, definition.count, `${split} row count`);
assert(splitRows.every((row) => row.split === split), `${split} contains a mismatched split value`);
rows.push(...splitRows);
}
const scenarios = await readJsonl("source/scenarios.jsonl");
assert.equal(rows.length, 1200, "total row count");
assert.equal(scenarios.length, 50, "scenario count");
const ids = new Set();
for (const [index, row] of rows.entries()) {
for (const field of requiredFields) {
assert(Object.hasOwn(row, field), `row ${index + 1} is missing ${field}`);
}
assert(!ids.has(row.id), `duplicate row id: ${row.id}`);
ids.add(row.id);
assert([0, 1].includes(row.label), `${row.id} has an invalid label`);
assert.equal(row.language, "en", `${row.id} is not English`);
assert.equal(row.source_type, "synthetic_curated", `${row.id} has an unexpected source type`);
assert(row.text.trim().length > 0, `${row.id} has empty text`);
}
const labelGroups = groupBy(rows, "label");
assert.equal(labelGroups.get(0)?.length, 600, "benign row count");
assert.equal(labelGroups.get(1)?.length, 600, "attack row count");
const pairs = groupBy(rows, "pair_id");
assert.equal(pairs.size, 600, "pair count");
for (const [pairId, pairRows] of pairs) {
assert.equal(pairRows.length, 2, `${pairId} must contain two rows`);
assert.deepEqual(
[...pairRows.map((row) => row.label)].sort(),
[0, 1],
`${pairId} must contain one benign and one attack row`,
);
for (const field of [
"scenario_id",
"split",
"pair_family",
"source_context",
"risk_domain",
"target_boundary",
]) {
assert.equal(new Set(pairRows.map((row) => row[field])).size, 1, `${pairId} mismatches ${field}`);
}
const attack = pairRows.find((row) => row.label === 1);
assert.equal(attack.attack_family, attack.pair_family, `${pairId} attack family mismatch`);
}
const scenarioIds = new Set(scenarios.map((scenario) => scenario.scenario_id));
assert.equal(scenarioIds.size, 50, "scenario identifiers must be unique");
const rowsByScenario = groupBy(rows, "scenario_id");
assert.equal(rowsByScenario.size, 50, "dataset scenario count");
for (const [scenarioId, scenarioRows] of rowsByScenario) {
assert(scenarioIds.has(scenarioId), `${scenarioId} is missing from scenarios.jsonl`);
assert.equal(scenarioRows.length, 24, `${scenarioId} row count`);
assert.equal(new Set(scenarioRows.map((row) => row.pair_id)).size, 12, `${scenarioId} pair count`);
assert.equal(new Set(scenarioRows.map((row) => row.pair_family)).size, 12, `${scenarioId} family count`);
assert.equal(new Set(scenarioRows.map((row) => row.split)).size, 1, `${scenarioId} crosses splits`);
}
const attackFamilies = groupBy(rows.filter((row) => row.label === 1), "attack_family");
assert.equal(attackFamilies.size, 12, "attack family count");
for (const [family, familyRows] of attackFamilies) {
assert.equal(familyRows.length, 50, `${family} attack count`);
}
const html = await readFile(resolve(root, "index.html"), "utf8");
for (const id of expectedUiIds) {
assert(html.includes(`id="${id}"`), `index.html is missing #${id}`);
}
assert(html.includes('src="assets/app.js"'), "index.html does not load app.js");
assert(html.includes('href="assets/styles.css"'), "index.html does not load styles.css");
console.log("Agentic Explorer validation passed");
console.log(" 1,200 rows | 600 pairs | 50 scenarios | 12 attack families");
console.log(" train 840 | validation 120 | test 240 | labels 600/600");
|