Ferrell Synthetic Intelligence commited on
Commit
61b024b
·
1 Parent(s): e41d0be

Add executable Veritas verification gate

Browse files
README.md CHANGED
@@ -39,6 +39,8 @@ See `runtime/README.md` and `models/manifest.json` for the adapter contract and
39
 
40
  The original AIDE Developer's Credo, inspired by broad Mandalorian discipline themes and translated into engineering behavior, lives in `harness/credo.md`; its research and copyright boundary is in `harness/credo-research.md`. It is mandatory harness context for every model role, alongside the role-specific SOP cards in `harness/sops.json`. `harness/veritas.mjs` uses calibrated task thresholds: 90% for ordinary explanation/code-change evidence and 98% for security, publishing, payment, and identity operations. These are evidence gates, not promises of universal model accuracy; failed gates produce abstention.
41
 
 
 
42
  ## Status
43
 
44
  This is a pre-production engineering release. The Liquid checkpoint is not included until its exact artifact, license, checksum, and evaluation are confirmed. The Qwen weight is downloaded separately from its official repository and should be verified before offline use.
 
39
 
40
  The original AIDE Developer's Credo, inspired by broad Mandalorian discipline themes and translated into engineering behavior, lives in `harness/credo.md`; its research and copyright boundary is in `harness/credo-research.md`. It is mandatory harness context for every model role, alongside the role-specific SOP cards in `harness/sops.json`. `harness/veritas.mjs` uses calibrated task thresholds: 90% for ordinary explanation/code-change evidence and 98% for security, publishing, payment, and identity operations. These are evidence gates, not promises of universal model accuracy; failed gates produce abstention.
41
 
42
+ `npm run veritas` executes the real local gate before a release or verified answer: compile checks, tests, Git whitespace checks, manifest validation, path boundaries, and secret scanning. It is intentionally allowlisted and does not execute arbitrary model-generated commands.
43
+
44
  ## Status
45
 
46
  This is a pre-production engineering release. The Liquid checkpoint is not included until its exact artifact, license, checksum, and evaluation are confirmed. The Qwen weight is downloaded separately from its official repository and should be verified before offline use.
harness/README.md CHANGED
@@ -36,4 +36,6 @@ intake -> guard -> retrieve -> plan -> propose -> verify -> revise -> test -> re
36
 
37
  See `orchestrator.mjs` and `policy.json` for the executable contract.
38
 
 
 
39
  The research and translation boundary is documented in `credo-research.md`. The product uses original engineering language rather than fictional quotations or branding.
 
36
 
37
  See `orchestrator.mjs` and `policy.json` for the executable contract.
38
 
39
+ Run `npm run veritas` from a workspace to execute the allowlisted compile, test, Git diff, manifest, secret, and path-boundary checks. A model verdict never overrides a failed execution check. The orchestrator accepts this runner as `verificationRunner` and blocks the final status until it passes.
40
+
41
  The research and translation boundary is documented in `credo-research.md`. The product uses original engineering language rather than fictional quotations or branding.
harness/checks.mjs ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { execFile } from 'node:child_process';
2
+ import { promises as fs } from 'node:fs';
3
+ import path from 'node:path';
4
+
5
+ const ALLOWED_COMMANDS = Object.freeze({
6
+ compile: ['npm', ['run', 'check']],
7
+ tests: ['npm', ['test']],
8
+ 'git-diff': ['git', ['diff', '--check']]
9
+ });
10
+
11
+ function command(name, cwd) {
12
+ const entry = ALLOWED_COMMANDS[name];
13
+ if (!entry) throw new Error(`command is not allowlisted: ${name}`);
14
+ return new Promise(resolve => {
15
+ execFile(entry[0], entry[1], { cwd, timeout: 120000, maxBuffer: 512 * 1024 }, (error, stdout, stderr) => {
16
+ resolve({
17
+ name,
18
+ passed: !error,
19
+ exit_code: error?.code ?? 0,
20
+ output: `${stdout}${stderr}`.slice(-12000)
21
+ });
22
+ });
23
+ });
24
+ }
25
+
26
+ async function secretScan(workspace) {
27
+ const suspicious = /(hf_[A-Za-z0-9]{20,}|sk-[A-Za-z0-9]{20,}|BEGIN (RSA|OPENSSH|EC) PRIVATE KEY)/;
28
+ const files = [];
29
+ async function walk(directory, depth = 0) {
30
+ if (depth > 4) return;
31
+ for (const entry of await fs.readdir(directory, { withFileTypes: true })) {
32
+ if (entry.name.startsWith('.') || entry.name === 'node_modules') continue;
33
+ const target = path.join(directory, entry.name);
34
+ if (entry.isDirectory()) await walk(target, depth + 1);
35
+ else if (entry.isFile() && (entry.name.endsWith('.js') || entry.name.endsWith('.mjs') || entry.name.endsWith('.json') || entry.name.endsWith('.md') || entry.name.endsWith('.html'))) files.push(target);
36
+ }
37
+ }
38
+ await walk(workspace);
39
+ const hits = [];
40
+ for (const file of files) {
41
+ const content = await fs.readFile(file, 'utf8');
42
+ if (suspicious.test(content)) hits.push(path.relative(workspace, file));
43
+ }
44
+ return { name: 'secret-scan', passed: hits.length === 0, hits };
45
+ }
46
+
47
+ async function manifestCheck(workspace) {
48
+ const files = ['models/manifest.json', 'community/node-manifest.json', 'release/package-manifest.json'];
49
+ const errors = [];
50
+ for (const file of files) {
51
+ try { JSON.parse(await fs.readFile(path.join(workspace, file), 'utf8')); }
52
+ catch (error) { errors.push(`${file}: ${error.message}`); }
53
+ }
54
+ return { name: 'manifest-validation', passed: errors.length === 0, errors };
55
+ }
56
+
57
+ function boundaryCheck(workspace, changedFiles = []) {
58
+ const root = path.resolve(workspace) + path.sep;
59
+ const invalid = changedFiles.filter(file => !path.resolve(workspace, file).startsWith(root));
60
+ return { name: 'path-boundary', passed: invalid.length === 0, invalid };
61
+ }
62
+
63
+ export async function runVeritasChecks({ workspace, changedFiles = [] } = {}) {
64
+ const results = [boundaryCheck(workspace, changedFiles), await secretScan(workspace), await manifestCheck(workspace)];
65
+ for (const name of ['compile', 'tests', 'git-diff']) results.push(await command(name, workspace));
66
+ return {
67
+ passed: results.every(result => result.passed),
68
+ checks: Object.fromEntries(results.map(result => [result.name, result.passed])),
69
+ results
70
+ };
71
+ }
harness/orchestrator.mjs CHANGED
@@ -1,4 +1,4 @@
1
- import { evaluateVeritas } from './veritas.mjs';
2
 
3
  const DEFAULT_POLICY = Object.freeze({
4
  max_turns: 4,
@@ -32,7 +32,7 @@ function bounded(value, max, label) {
32
  return text;
33
  }
34
 
35
- export function createHarness({ providers, tools = {}, policy = {} }) {
36
  const rules = { ...DEFAULT_POLICY, ...policy };
37
  return {
38
  async run(task, context = {}) {
@@ -71,11 +71,10 @@ export function createHarness({ providers, tools = {}, policy = {} }) {
71
  patch
72
  });
73
  const approved = /^\s*APPROVE\b/i.test(String(verdict));
74
- const veritas = evaluateVeritas({
75
- taskClass: context.taskClass || 'code-change',
76
- evidenceScore: context.evidenceScore,
77
- checks: context.checks
78
- });
79
  trace.push({ stage: 'verify', status: approved ? 'approved-with-notes' : 'blocked' });
80
  trace.push({ stage: 'veritas', status: veritas.status, score: veritas.score, threshold: veritas.threshold });
81
  return {
@@ -83,6 +82,7 @@ export function createHarness({ providers, tools = {}, policy = {} }) {
83
  plan,
84
  patch,
85
  verdict,
 
86
  veritas,
87
  trace,
88
  apply: async () => {
 
1
+ import { evaluateExecution, evaluateVeritas } from './veritas.mjs';
2
 
3
  const DEFAULT_POLICY = Object.freeze({
4
  max_turns: 4,
 
32
  return text;
33
  }
34
 
35
+ export function createHarness({ providers, tools = {}, policy = {}, verificationRunner = null }) {
36
  const rules = { ...DEFAULT_POLICY, ...policy };
37
  return {
38
  async run(task, context = {}) {
 
71
  patch
72
  });
73
  const approved = /^\s*APPROVE\b/i.test(String(verdict));
74
+ const execution = verificationRunner ? await verificationRunner({ task: goal, plan, patch, context }) : null;
75
+ const veritas = execution
76
+ ? evaluateExecution({ taskClass: context.taskClass || 'code-change', execution })
77
+ : evaluateVeritas({ taskClass: context.taskClass || 'code-change', evidenceScore: context.evidenceScore, checks: context.checks });
 
78
  trace.push({ stage: 'verify', status: approved ? 'approved-with-notes' : 'blocked' });
79
  trace.push({ stage: 'veritas', status: veritas.status, score: veritas.score, threshold: veritas.threshold });
80
  return {
 
82
  plan,
83
  patch,
84
  verdict,
85
+ execution,
86
  veritas,
87
  trace,
88
  apply: async () => {
harness/run-veritas.mjs ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ import path from 'node:path';
2
+ import { fileURLToPath } from 'node:url';
3
+ import { runVeritasChecks } from './checks.mjs';
4
+
5
+ const root = path.resolve(process.argv[2] || path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'));
6
+ const result = await runVeritasChecks({ workspace: root });
7
+ console.log(JSON.stringify(result, null, 2));
8
+ if (!result.passed) process.exitCode = 1;
harness/test-orchestrator.mjs CHANGED
@@ -17,4 +17,12 @@ assert.equal(calls.length, 3);
17
  assert.equal(result.veritas.status, 'abstain-needs-evidence');
18
  assert.ok(calls.every(call => call.input.mandatory_credo.includes('Protect the user')));
19
  await assert.rejects(result.apply(), /permission-gated daemon/);
 
 
 
 
 
 
 
 
20
  console.log('universal harness test passed');
 
17
  assert.equal(result.veritas.status, 'abstain-needs-evidence');
18
  assert.ok(calls.every(call => call.input.mandatory_credo.includes('Protect the user')));
19
  await assert.rejects(result.apply(), /permission-gated daemon/);
20
+ const verifiedHarness = createHarness({
21
+ providers: { reason: provider('reason'), build: provider('build'), verify: provider('verify') },
22
+ policy: { require_human_approval: false },
23
+ verificationRunner: async () => ({ passed: true, checks: { compile: true, tests: true, 'git-diff': true } })
24
+ });
25
+ const verified = await verifiedHarness.run('Run the verified flow', { taskClass: 'code-change' });
26
+ assert.equal(verified.status, 'ready-for-apply');
27
+ assert.equal(verified.veritas.status, 'verified');
28
  console.log('universal harness test passed');
harness/veritas.mjs CHANGED
@@ -19,3 +19,8 @@ export function evaluateVeritas({ taskClass = 'code-change', evidenceScore = 0,
19
  rule: 'Model confidence is not evidence. A failed deterministic gate blocks final output.'
20
  };
21
  }
 
 
 
 
 
 
19
  rule: 'Model confidence is not evidence. A failed deterministic gate blocks final output.'
20
  };
21
  }
22
+
23
+ export function evaluateExecution({ taskClass = 'code-change', execution } = {}) {
24
+ const evidenceScore = execution?.passed ? 1 : 0;
25
+ return evaluateVeritas({ taskClass, evidenceScore, checks: execution?.checks || {} });
26
+ }
package.json CHANGED
@@ -6,7 +6,8 @@
6
  "type": "module",
7
  "scripts": {
8
  "test": "node tests/smoke.mjs && node harness/test-orchestrator.mjs",
9
- "check": "node --check app.js && node --check daemon/server.mjs"
 
10
  },
11
  "engines": {
12
  "node": ">=20"
 
6
  "type": "module",
7
  "scripts": {
8
  "test": "node tests/smoke.mjs && node harness/test-orchestrator.mjs",
9
+ "check": "node --check app.js && node --check daemon/server.mjs && node --check harness/orchestrator.mjs && node --check harness/checks.mjs",
10
+ "veritas": "node harness/run-veritas.mjs"
11
  },
12
  "engines": {
13
  "node": ">=20"