Spaces:
Running
Running
| ; | |
| const test = require('node:test'); | |
| const assert = require('node:assert/strict'); | |
| const fs = require('node:fs'); | |
| const path = require('node:path'); | |
| const zlib = require('node:zlib'); | |
| const vm = require('node:vm'); | |
| const { validateObservatoryTrace } = require('../trace-validation.js'); | |
| const root = path.resolve(__dirname, '..'); | |
| const models = JSON.parse(fs.readFileSync(path.join(root, 'models.json'), 'utf8')); | |
| const readTrace = id => JSON.parse(zlib.gunzipSync(fs.readFileSync(path.join(root, 'traces', id + '-sky-q8.json.gz')))); | |
| const captures = { flint: readTrace('flint'), qwen: readTrace('qwen') }; | |
| // Invalid-input tests change small copies of real captured frames. The distributed | |
| // captures remain untouched, and these changes are never presented as activations. | |
| function sample(id = 'flint') { | |
| const original = captures[id]; | |
| return structuredClone({ ...original, frames: original.frames.slice(0, 2) }); | |
| } | |
| function rejects(title, mutate, id = 'flint') { | |
| test(title, () => { | |
| const trace = sample(id); | |
| mutate(trace); | |
| assert.throws(() => validateObservatoryTrace(trace, models), Error); | |
| }); | |
| } | |
| for (const id of ['flint', 'qwen']) { | |
| test('accepts every real ' + id + ' frame without changing the source', () => { | |
| const trace = captures[id]; | |
| const before = JSON.stringify(trace); | |
| assert.equal(validateObservatoryTrace(trace, models), models.find(model => model.id === id)); | |
| assert.equal(JSON.stringify(trace), before); | |
| }); | |
| } | |
| test('accepts legacy Flint traces without model metadata or token bytes', () => { | |
| const trace = sample(); | |
| delete trace.model; | |
| for (const frame of trace.frames) { delete frame.model; delete frame.token_bytes; } | |
| assert.equal(validateObservatoryTrace(trace, models).id, 'flint'); | |
| }); | |
| test('accepts a stopped capture that retains its measured frames', () => { | |
| const trace = sample(); | |
| trace.result.reason = 'stopped'; | |
| assert.equal(validateObservatoryTrace(trace, models).id, 'flint'); | |
| }); | |
| test('accepts floating point probability roundoff within capture precision', () => { | |
| const trace = sample('qwen'); | |
| trace.frames[0].layers[0].retention[0] = 1 + 0.000001; | |
| assert.equal(validateObservatoryTrace(trace, models).id, 'qwen'); | |
| }); | |
| test('exports the browser validator without requiring Node globals', () => { | |
| const context = {}; | |
| vm.runInNewContext(fs.readFileSync(path.join(root, 'trace-validation.js'), 'utf8'), context); | |
| assert.equal(context.validateObservatoryTrace(sample(), models).id, 'flint'); | |
| }); | |
| rejects('rejects model metadata that contradicts the schema', t => { t.model.id = 'qwen'; }); | |
| rejects('rejects unrecognized trace schema', t => { t.schema = 'arbitrary.v1'; }); | |
| rejects('rejects empty captures', t => { t.frames = []; }); | |
| rejects('rejects excessive frame counts', t => { t.frames = Array(257).fill(t.frames[0]); }); | |
| rejects('rejects wrong full-attention dimensions', t => { t.architecture.head_dim++; }); | |
| rejects('rejects mismatched expert pool dimensions', t => { t.architecture.num_experts = 128; }); | |
| rejects('rejects mismatched expert feed-forward width', t => { t.architecture.moe_intermediate_size++; }); | |
| rejects('rejects changed dense layer assignments', t => { t.architecture.mlp_only_layers = [0]; }); | |
| rejects('rejects changed sliding-window size', t => { t.architecture.sliding_window++; }); | |
| rejects('rejects recurrent key-head dimension mismatch', t => { t.architecture.linear_key_head_dim++; }, 'qwen'); | |
| rejects('rejects recurrent value-head count mismatch', t => { t.architecture.linear_num_value_heads++; }, 'qwen'); | |
| rejects('rejects quantization mismatches', t => { t.model.quantization = 'Q4_0'; }); | |
| rejects('rejects malformed prompt tokens', t => { t.start.tokens[0] = {}; }); | |
| rejects('rejects excessive prompt-token counts', t => { t.start.tokens = Array(2049).fill('x'); }); | |
| rejects('rejects oversized individual token strings', t => { t.start.tokens[0] = 'x'.repeat(8193); }); | |
| rejects('rejects inconsistent declared prompt length', t => { t.start.prompt_tokens++; }); | |
| rejects('rejects oversized prompts', t => { t.prompt = 'x'.repeat(4001); }); | |
| rejects('rejects invalid generation settings', t => { t.settings.seed = -1; }); | |
| rejects('rejects non-boolean reasoning settings', t => { t.settings.reasoning = 'false'; }); | |
| rejects('rejects invalid temperature', t => { t.settings.temperature = Infinity; }); | |
| rejects('rejects model changes inside captured frames', t => { t.frames[0].model = 'qwen'; }); | |
| rejects('rejects inconsistent frame positions', t => { t.frames[0].input_position++; }); | |
| rejects('rejects reordered token steps', t => { t.frames[1].step = 0; }); | |
| rejects('rejects negative token IDs', t => { t.frames[0].token_id = -1; }); | |
| rejects('rejects out-of-vocabulary token IDs', t => { t.frames[0].token_id = t.architecture.vocab_size; }); | |
| rejects('rejects invalid raw token bytes', t => { t.frames[0].token_bytes = [256]; }); | |
| rejects('rejects fractional raw token bytes', t => { t.frames[0].token_bytes = [1.5]; }); | |
| rejects('rejects null raw token byte arrays', t => { t.frames[0].token_bytes = null; }); | |
| rejects('rejects nonfinite timings', t => { t.frames[0].duration = NaN; }); | |
| rejects('rejects elapsed time moving backwards', t => { t.frames[1].elapsed = 0; }); | |
| rejects('rejects missing layers', t => { t.frames[0].layers.pop(); }); | |
| rejects('rejects missing head measurements', t => { t.frames[0].layers[0].heads.pop(); }); | |
| rejects('rejects NaN output magnitudes', t => { t.frames[0].layers[0].heads[0] = NaN; }); | |
| rejects('rejects infinite residual output', t => { t.frames[0].layers[0].residual = Infinity; }); | |
| rejects('rejects negative feed-forward output', t => { t.frames[0].layers[0].ffn = -1; }); | |
| rejects('rejects inconsistent layer indices', t => { t.frames[0].layers[0].index = 1; }); | |
| rejects('rejects repeated expert selections', t => { t.frames[0].layers[2].experts[0] = t.frames[0].layers[2].experts[1]; }); | |
| rejects('rejects expert IDs outside the pool', t => { t.frames[0].layers[2].experts[0] = 256; }); | |
| rejects('rejects non-normalized expert weights', t => { t.frames[0].layers[2].weights[0] = 0; }); | |
| rejects('rejects missing shared expert measurements', t => { delete t.frames[0].layers[2].shared; }); | |
| rejects('rejects nonfinite routed output', t => { t.frames[0].layers[2].routed = Infinity; }); | |
| rejects('rejects missing recurrent state measurements', t => { delete t.frames[0].layers[0].state_rms; }, 'qwen'); | |
| rejects('rejects invalid recurrent retention probability', t => { t.frames[0].layers[0].retention[0] = 1.01; }, 'qwen'); | |
| rejects('rejects negative recurrent write gate', t => { t.frames[0].layers[0].write_gate[0] = -0.1; }, 'qwen'); | |
| rejects('rejects wrong attention head count', t => { t.frames[0].layers[0].attention.pop(); }); | |
| rejects('rejects invalid attention entropy', t => { t.frames[0].layers[0].attention[0].entropy = Infinity; }); | |
| rejects('rejects invalid learned sink probability', t => { t.frames[0].layers[0].attention[0].sink = 1.1; }); | |
| rejects('rejects attention to positions in the future', t => { t.frames[0].layers[0].attention[0].top[0][0] = 2047; }); | |
| rejects('rejects repeated attention positions', t => { const a = t.frames[0].layers[0].attention[0]; a.top[0][0] = a.top[1][0]; }); | |
| rejects('rejects attention probability mass exceeding one', t => { t.frames[0].layers[0].attention[0].top[0][1] = 1; }); | |
| test('failed validation does not change a previously selected capture', () => { | |
| const loaded = { model: models[0], trace: captures.flint }; | |
| const previous = { ...loaded }; | |
| const candidate = sample('qwen'); | |
| candidate.frames[0].layers[0].heads = []; | |
| assert.throws(() => { | |
| const model = validateObservatoryTrace(candidate, models); | |
| loaded.model = model; | |
| loaded.trace = candidate; | |
| }); | |
| assert.equal(loaded.model, previous.model); | |
| assert.equal(loaded.trace, previous.trace); | |
| }); | |