Ferrell Synthetic Intelligence commited on
Commit ·
61b024b
1
Parent(s): e41d0be
Add executable Veritas verification gate
Browse files- README.md +2 -0
- harness/README.md +2 -0
- harness/checks.mjs +71 -0
- harness/orchestrator.mjs +7 -7
- harness/run-veritas.mjs +8 -0
- harness/test-orchestrator.mjs +8 -0
- harness/veritas.mjs +5 -0
- package.json +2 -1
README.md
CHANGED
|
@@ -39,6 +39,8 @@ See `runtime/README.md` and `models/manifest.json` for the adapter contract and
|
|
| 39 |
|
| 40 |
The original AIDE Developer's Credo, inspired by broad Mandalorian discipline themes and translated into engineering behavior, lives in `harness/credo.md`; its research and copyright boundary is in `harness/credo-research.md`. It is mandatory harness context for every model role, alongside the role-specific SOP cards in `harness/sops.json`. `harness/veritas.mjs` uses calibrated task thresholds: 90% for ordinary explanation/code-change evidence and 98% for security, publishing, payment, and identity operations. These are evidence gates, not promises of universal model accuracy; failed gates produce abstention.
|
| 41 |
|
|
|
|
|
|
|
| 42 |
## Status
|
| 43 |
|
| 44 |
This is a pre-production engineering release. The Liquid checkpoint is not included until its exact artifact, license, checksum, and evaluation are confirmed. The Qwen weight is downloaded separately from its official repository and should be verified before offline use.
|
|
|
|
| 39 |
|
| 40 |
The original AIDE Developer's Credo, inspired by broad Mandalorian discipline themes and translated into engineering behavior, lives in `harness/credo.md`; its research and copyright boundary is in `harness/credo-research.md`. It is mandatory harness context for every model role, alongside the role-specific SOP cards in `harness/sops.json`. `harness/veritas.mjs` uses calibrated task thresholds: 90% for ordinary explanation/code-change evidence and 98% for security, publishing, payment, and identity operations. These are evidence gates, not promises of universal model accuracy; failed gates produce abstention.
|
| 41 |
|
| 42 |
+
`npm run veritas` executes the real local gate before a release or verified answer: compile checks, tests, Git whitespace checks, manifest validation, path boundaries, and secret scanning. It is intentionally allowlisted and does not execute arbitrary model-generated commands.
|
| 43 |
+
|
| 44 |
## Status
|
| 45 |
|
| 46 |
This is a pre-production engineering release. The Liquid checkpoint is not included until its exact artifact, license, checksum, and evaluation are confirmed. The Qwen weight is downloaded separately from its official repository and should be verified before offline use.
|
harness/README.md
CHANGED
|
@@ -36,4 +36,6 @@ intake -> guard -> retrieve -> plan -> propose -> verify -> revise -> test -> re
|
|
| 36 |
|
| 37 |
See `orchestrator.mjs` and `policy.json` for the executable contract.
|
| 38 |
|
|
|
|
|
|
|
| 39 |
The research and translation boundary is documented in `credo-research.md`. The product uses original engineering language rather than fictional quotations or branding.
|
|
|
|
| 36 |
|
| 37 |
See `orchestrator.mjs` and `policy.json` for the executable contract.
|
| 38 |
|
| 39 |
+
Run `npm run veritas` from a workspace to execute the allowlisted compile, test, Git diff, manifest, secret, and path-boundary checks. A model verdict never overrides a failed execution check. The orchestrator accepts this runner as `verificationRunner` and blocks the final status until it passes.
|
| 40 |
+
|
| 41 |
The research and translation boundary is documented in `credo-research.md`. The product uses original engineering language rather than fictional quotations or branding.
|
harness/checks.mjs
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { execFile } from 'node:child_process';
|
| 2 |
+
import { promises as fs } from 'node:fs';
|
| 3 |
+
import path from 'node:path';
|
| 4 |
+
|
| 5 |
+
const ALLOWED_COMMANDS = Object.freeze({
|
| 6 |
+
compile: ['npm', ['run', 'check']],
|
| 7 |
+
tests: ['npm', ['test']],
|
| 8 |
+
'git-diff': ['git', ['diff', '--check']]
|
| 9 |
+
});
|
| 10 |
+
|
| 11 |
+
function command(name, cwd) {
|
| 12 |
+
const entry = ALLOWED_COMMANDS[name];
|
| 13 |
+
if (!entry) throw new Error(`command is not allowlisted: ${name}`);
|
| 14 |
+
return new Promise(resolve => {
|
| 15 |
+
execFile(entry[0], entry[1], { cwd, timeout: 120000, maxBuffer: 512 * 1024 }, (error, stdout, stderr) => {
|
| 16 |
+
resolve({
|
| 17 |
+
name,
|
| 18 |
+
passed: !error,
|
| 19 |
+
exit_code: error?.code ?? 0,
|
| 20 |
+
output: `${stdout}${stderr}`.slice(-12000)
|
| 21 |
+
});
|
| 22 |
+
});
|
| 23 |
+
});
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
async function secretScan(workspace) {
|
| 27 |
+
const suspicious = /(hf_[A-Za-z0-9]{20,}|sk-[A-Za-z0-9]{20,}|BEGIN (RSA|OPENSSH|EC) PRIVATE KEY)/;
|
| 28 |
+
const files = [];
|
| 29 |
+
async function walk(directory, depth = 0) {
|
| 30 |
+
if (depth > 4) return;
|
| 31 |
+
for (const entry of await fs.readdir(directory, { withFileTypes: true })) {
|
| 32 |
+
if (entry.name.startsWith('.') || entry.name === 'node_modules') continue;
|
| 33 |
+
const target = path.join(directory, entry.name);
|
| 34 |
+
if (entry.isDirectory()) await walk(target, depth + 1);
|
| 35 |
+
else if (entry.isFile() && (entry.name.endsWith('.js') || entry.name.endsWith('.mjs') || entry.name.endsWith('.json') || entry.name.endsWith('.md') || entry.name.endsWith('.html'))) files.push(target);
|
| 36 |
+
}
|
| 37 |
+
}
|
| 38 |
+
await walk(workspace);
|
| 39 |
+
const hits = [];
|
| 40 |
+
for (const file of files) {
|
| 41 |
+
const content = await fs.readFile(file, 'utf8');
|
| 42 |
+
if (suspicious.test(content)) hits.push(path.relative(workspace, file));
|
| 43 |
+
}
|
| 44 |
+
return { name: 'secret-scan', passed: hits.length === 0, hits };
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
async function manifestCheck(workspace) {
|
| 48 |
+
const files = ['models/manifest.json', 'community/node-manifest.json', 'release/package-manifest.json'];
|
| 49 |
+
const errors = [];
|
| 50 |
+
for (const file of files) {
|
| 51 |
+
try { JSON.parse(await fs.readFile(path.join(workspace, file), 'utf8')); }
|
| 52 |
+
catch (error) { errors.push(`${file}: ${error.message}`); }
|
| 53 |
+
}
|
| 54 |
+
return { name: 'manifest-validation', passed: errors.length === 0, errors };
|
| 55 |
+
}
|
| 56 |
+
|
| 57 |
+
function boundaryCheck(workspace, changedFiles = []) {
|
| 58 |
+
const root = path.resolve(workspace) + path.sep;
|
| 59 |
+
const invalid = changedFiles.filter(file => !path.resolve(workspace, file).startsWith(root));
|
| 60 |
+
return { name: 'path-boundary', passed: invalid.length === 0, invalid };
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
export async function runVeritasChecks({ workspace, changedFiles = [] } = {}) {
|
| 64 |
+
const results = [boundaryCheck(workspace, changedFiles), await secretScan(workspace), await manifestCheck(workspace)];
|
| 65 |
+
for (const name of ['compile', 'tests', 'git-diff']) results.push(await command(name, workspace));
|
| 66 |
+
return {
|
| 67 |
+
passed: results.every(result => result.passed),
|
| 68 |
+
checks: Object.fromEntries(results.map(result => [result.name, result.passed])),
|
| 69 |
+
results
|
| 70 |
+
};
|
| 71 |
+
}
|
harness/orchestrator.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
import { evaluateVeritas } from './veritas.mjs';
|
| 2 |
|
| 3 |
const DEFAULT_POLICY = Object.freeze({
|
| 4 |
max_turns: 4,
|
|
@@ -32,7 +32,7 @@ function bounded(value, max, label) {
|
|
| 32 |
return text;
|
| 33 |
}
|
| 34 |
|
| 35 |
-
export function createHarness({ providers, tools = {}, policy = {} }) {
|
| 36 |
const rules = { ...DEFAULT_POLICY, ...policy };
|
| 37 |
return {
|
| 38 |
async run(task, context = {}) {
|
|
@@ -71,11 +71,10 @@ export function createHarness({ providers, tools = {}, policy = {} }) {
|
|
| 71 |
patch
|
| 72 |
});
|
| 73 |
const approved = /^\s*APPROVE\b/i.test(String(verdict));
|
| 74 |
-
const
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
checks: context.checks
|
| 78 |
-
});
|
| 79 |
trace.push({ stage: 'verify', status: approved ? 'approved-with-notes' : 'blocked' });
|
| 80 |
trace.push({ stage: 'veritas', status: veritas.status, score: veritas.score, threshold: veritas.threshold });
|
| 81 |
return {
|
|
@@ -83,6 +82,7 @@ export function createHarness({ providers, tools = {}, policy = {} }) {
|
|
| 83 |
plan,
|
| 84 |
patch,
|
| 85 |
verdict,
|
|
|
|
| 86 |
veritas,
|
| 87 |
trace,
|
| 88 |
apply: async () => {
|
|
|
|
| 1 |
+
import { evaluateExecution, evaluateVeritas } from './veritas.mjs';
|
| 2 |
|
| 3 |
const DEFAULT_POLICY = Object.freeze({
|
| 4 |
max_turns: 4,
|
|
|
|
| 32 |
return text;
|
| 33 |
}
|
| 34 |
|
| 35 |
+
export function createHarness({ providers, tools = {}, policy = {}, verificationRunner = null }) {
|
| 36 |
const rules = { ...DEFAULT_POLICY, ...policy };
|
| 37 |
return {
|
| 38 |
async run(task, context = {}) {
|
|
|
|
| 71 |
patch
|
| 72 |
});
|
| 73 |
const approved = /^\s*APPROVE\b/i.test(String(verdict));
|
| 74 |
+
const execution = verificationRunner ? await verificationRunner({ task: goal, plan, patch, context }) : null;
|
| 75 |
+
const veritas = execution
|
| 76 |
+
? evaluateExecution({ taskClass: context.taskClass || 'code-change', execution })
|
| 77 |
+
: evaluateVeritas({ taskClass: context.taskClass || 'code-change', evidenceScore: context.evidenceScore, checks: context.checks });
|
|
|
|
| 78 |
trace.push({ stage: 'verify', status: approved ? 'approved-with-notes' : 'blocked' });
|
| 79 |
trace.push({ stage: 'veritas', status: veritas.status, score: veritas.score, threshold: veritas.threshold });
|
| 80 |
return {
|
|
|
|
| 82 |
plan,
|
| 83 |
patch,
|
| 84 |
verdict,
|
| 85 |
+
execution,
|
| 86 |
veritas,
|
| 87 |
trace,
|
| 88 |
apply: async () => {
|
harness/run-veritas.mjs
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import path from 'node:path';
|
| 2 |
+
import { fileURLToPath } from 'node:url';
|
| 3 |
+
import { runVeritasChecks } from './checks.mjs';
|
| 4 |
+
|
| 5 |
+
const root = path.resolve(process.argv[2] || path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'));
|
| 6 |
+
const result = await runVeritasChecks({ workspace: root });
|
| 7 |
+
console.log(JSON.stringify(result, null, 2));
|
| 8 |
+
if (!result.passed) process.exitCode = 1;
|
harness/test-orchestrator.mjs
CHANGED
|
@@ -17,4 +17,12 @@ assert.equal(calls.length, 3);
|
|
| 17 |
assert.equal(result.veritas.status, 'abstain-needs-evidence');
|
| 18 |
assert.ok(calls.every(call => call.input.mandatory_credo.includes('Protect the user')));
|
| 19 |
await assert.rejects(result.apply(), /permission-gated daemon/);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
console.log('universal harness test passed');
|
|
|
|
| 17 |
assert.equal(result.veritas.status, 'abstain-needs-evidence');
|
| 18 |
assert.ok(calls.every(call => call.input.mandatory_credo.includes('Protect the user')));
|
| 19 |
await assert.rejects(result.apply(), /permission-gated daemon/);
|
| 20 |
+
const verifiedHarness = createHarness({
|
| 21 |
+
providers: { reason: provider('reason'), build: provider('build'), verify: provider('verify') },
|
| 22 |
+
policy: { require_human_approval: false },
|
| 23 |
+
verificationRunner: async () => ({ passed: true, checks: { compile: true, tests: true, 'git-diff': true } })
|
| 24 |
+
});
|
| 25 |
+
const verified = await verifiedHarness.run('Run the verified flow', { taskClass: 'code-change' });
|
| 26 |
+
assert.equal(verified.status, 'ready-for-apply');
|
| 27 |
+
assert.equal(verified.veritas.status, 'verified');
|
| 28 |
console.log('universal harness test passed');
|
harness/veritas.mjs
CHANGED
|
@@ -19,3 +19,8 @@ export function evaluateVeritas({ taskClass = 'code-change', evidenceScore = 0,
|
|
| 19 |
rule: 'Model confidence is not evidence. A failed deterministic gate blocks final output.'
|
| 20 |
};
|
| 21 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
rule: 'Model confidence is not evidence. A failed deterministic gate blocks final output.'
|
| 20 |
};
|
| 21 |
}
|
| 22 |
+
|
| 23 |
+
export function evaluateExecution({ taskClass = 'code-change', execution } = {}) {
|
| 24 |
+
const evidenceScore = execution?.passed ? 1 : 0;
|
| 25 |
+
return evaluateVeritas({ taskClass, evidenceScore, checks: execution?.checks || {} });
|
| 26 |
+
}
|
package.json
CHANGED
|
@@ -6,7 +6,8 @@
|
|
| 6 |
"type": "module",
|
| 7 |
"scripts": {
|
| 8 |
"test": "node tests/smoke.mjs && node harness/test-orchestrator.mjs",
|
| 9 |
-
"check": "node --check app.js && node --check daemon/server.mjs"
|
|
|
|
| 10 |
},
|
| 11 |
"engines": {
|
| 12 |
"node": ">=20"
|
|
|
|
| 6 |
"type": "module",
|
| 7 |
"scripts": {
|
| 8 |
"test": "node tests/smoke.mjs && node harness/test-orchestrator.mjs",
|
| 9 |
+
"check": "node --check app.js && node --check daemon/server.mjs && node --check harness/orchestrator.mjs && node --check harness/checks.mjs",
|
| 10 |
+
"veritas": "node harness/run-veritas.mjs"
|
| 11 |
},
|
| 12 |
"engines": {
|
| 13 |
"node": ">=20"
|