import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { runSecretsAudit } from "./audit.js"; type AuditFixture = { rootDir: string; stateDir: string; configPath: string; authStorePath: string; authJsonPath: string; modelsPath: string; envPath: string; env: NodeJS.ProcessEnv; }; const OPENAI_API_KEY_MARKER = "OPENAI_API_KEY"; // pragma: allowlist secret const MAX_AUDIT_MODELS_JSON_BYTES = 5 * 1024 * 1024; async function writeJsonFile(filePath: string, value: unknown): Promise { await fs.writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8"); } function resolveRuntimePathEnv(): string { if (typeof process.env.PATH === "string" && process.env.PATH.trim().length > 0) { return process.env.PATH; } return "/usr/bin:/bin"; } function hasFinding( report: Awaited>, predicate: (entry: { code: string; file: string; jsonPath?: string }) => boolean, ): boolean { return report.findings.some((entry) => predicate(entry as { code: string; file: string; jsonPath?: string }), ); } async function createAuditFixture(): Promise { const rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-secrets-audit-")); const stateDir = path.join(rootDir, ".openclaw"); const configPath = path.join(stateDir, "openclaw.json"); const authStorePath = path.join(stateDir, "agents", "main", "agent", "auth-profiles.json"); const authJsonPath = path.join(stateDir, "agents", "main", "agent", "auth.json"); const modelsPath = path.join(stateDir, "agents", "main", "agent", "models.json"); const envPath = path.join(stateDir, ".env"); await fs.mkdir(path.dirname(configPath), { recursive: true }); await fs.mkdir(path.dirname(authStorePath), { recursive: true }); return { rootDir, stateDir, configPath, authStorePath, authJsonPath, modelsPath, envPath, env: { OPENCLAW_STATE_DIR: stateDir, OPENCLAW_CONFIG_PATH: configPath, OPENAI_API_KEY: "env-openai-key", // pragma: allowlist secret PATH: resolveRuntimePathEnv(), }, }; } async function seedAuditFixture(fixture: AuditFixture): Promise { const seededProvider = { openai: { baseUrl: "https://api.openai.com/v1", api: "openai-completions", apiKey: { source: "env", provider: "default", id: OPENAI_API_KEY_MARKER }, models: [{ id: "gpt-5", name: "gpt-5" }], }, }; const seededProfiles = new Map>([ [ "openai:default", { type: "api_key", provider: "openai", key: "sk-openai-plaintext", }, ], ]); await writeJsonFile(fixture.configPath, { models: { providers: seededProvider }, }); await writeJsonFile(fixture.authStorePath, { version: 1, profiles: Object.fromEntries(seededProfiles), }); await writeJsonFile(fixture.modelsPath, { providers: { openai: { baseUrl: "https://api.openai.com/v1", api: "openai-completions", apiKey: OPENAI_API_KEY_MARKER, models: [{ id: "gpt-5", name: "gpt-5" }], }, }, }); await fs.writeFile( fixture.envPath, `${OPENAI_API_KEY_MARKER}=sk-openai-plaintext\n`, // pragma: allowlist secret "utf8", ); } describe("secrets audit", () => { let fixture: AuditFixture; async function writeModelsProvider( overrides: Partial<{ apiKey: unknown; headers: Record; }> = {}, ) { await writeJsonFile(fixture.modelsPath, { providers: { openai: { baseUrl: "https://api.openai.com/v1", api: "openai-completions", apiKey: OPENAI_API_KEY_MARKER, models: [{ id: "gpt-5", name: "gpt-5" }], ...overrides, }, }, }); } function expectModelsFinding( report: Awaited>, params: { code: string; jsonPath?: string; present?: boolean }, ) { expect( hasFinding( report, (entry) => entry.code === params.code && entry.file === fixture.modelsPath && (params.jsonPath === undefined || entry.jsonPath === params.jsonPath), ), ).toBe(params.present ?? true); } beforeEach(async () => { fixture = await createAuditFixture(); await seedAuditFixture(fixture); }); afterEach(async () => { await fs.rm(fixture.rootDir, { recursive: true, force: true }); }); it("reports plaintext + shadowing findings", async () => { const report = await runSecretsAudit({ env: fixture.env }); expect(report.status).toBe("findings"); expect(report.summary.plaintextCount).toBeGreaterThan(0); expect(report.summary.shadowedRefCount).toBeGreaterThan(0); expect(hasFinding(report, (entry) => entry.code === "REF_SHADOWED")).toBe(true); expect(hasFinding(report, (entry) => entry.code === "PLAINTEXT_FOUND")).toBe(true); }); it("does not mutate legacy auth.json during audit", async () => { await fs.rm(fixture.authStorePath, { force: true }); await writeJsonFile(fixture.authJsonPath, { openai: { type: "api_key", key: "sk-legacy-auth-json", }, }); const report = await runSecretsAudit({ env: fixture.env }); expect(hasFinding(report, (entry) => entry.code === "LEGACY_RESIDUE")).toBe(true); await expect(fs.stat(fixture.authJsonPath)).resolves.toBeTruthy(); await expect(fs.stat(fixture.authStorePath)).rejects.toMatchObject({ code: "ENOENT" }); }); it("reports malformed sidecar JSON as findings instead of crashing", async () => { await fs.writeFile(fixture.authStorePath, "{invalid-json", "utf8"); await fs.writeFile(fixture.authJsonPath, "{invalid-json", "utf8"); const report = await runSecretsAudit({ env: fixture.env }); expect(hasFinding(report, (entry) => entry.file === fixture.authStorePath)).toBe(true); expect(hasFinding(report, (entry) => entry.file === fixture.authJsonPath)).toBe(true); expect(hasFinding(report, (entry) => entry.code === "REF_UNRESOLVED")).toBe(true); }); it("batches ref resolution per provider during audit", async () => { if (process.platform === "win32") { return; } const execLogPath = path.join(fixture.rootDir, "exec-calls.log"); const execScriptPath = path.join(fixture.rootDir, "resolver.sh"); await fs.writeFile( execScriptPath, [ "#!/bin/sh", `printf 'x\\n' >> ${JSON.stringify(execLogPath)}`, "cat >/dev/null", 'printf \'{"protocolVersion":1,"values":{"providers/openai/apiKey":"value:providers/openai/apiKey","providers/moonshot/apiKey":"value:providers/moonshot/apiKey"}}\'', // pragma: allowlist secret ].join("\n"), { encoding: "utf8", mode: 0o700 }, ); await writeJsonFile(fixture.configPath, { secrets: { providers: { execmain: { source: "exec", command: execScriptPath, jsonOnly: true, timeoutMs: 20_000, noOutputTimeoutMs: 10_000, }, }, }, models: { providers: { openai: { baseUrl: "https://api.openai.com/v1", api: "openai-completions", apiKey: { source: "exec", provider: "execmain", id: "providers/openai/apiKey" }, models: [{ id: "gpt-5", name: "gpt-5" }], }, moonshot: { baseUrl: "https://api.moonshot.cn/v1", api: "openai-completions", apiKey: { source: "exec", provider: "execmain", id: "providers/moonshot/apiKey" }, models: [{ id: "moonshot-v1-8k", name: "moonshot-v1-8k" }], }, }, }, }); await fs.rm(fixture.authStorePath, { force: true }); await fs.writeFile(fixture.envPath, "", "utf8"); const report = await runSecretsAudit({ env: fixture.env }); expect(report.summary.unresolvedRefCount).toBe(0); const callLog = await fs.readFile(execLogPath, "utf8"); const callCount = callLog.split("\n").filter((line) => line.trim().length > 0).length; expect(callCount).toBe(1); }); it("short-circuits per-ref fallback for provider-wide batch failures", async () => { if (process.platform === "win32") { return; } const execLogPath = path.join(fixture.rootDir, "exec-fail-calls.log"); const execScriptPath = path.join(fixture.rootDir, "resolver-fail.mjs"); await fs.writeFile( execScriptPath, [ "#!/usr/bin/env node", "import fs from 'node:fs';", `fs.appendFileSync(${JSON.stringify(execLogPath)}, 'x\\n');`, "process.exit(1);", ].join("\n"), { encoding: "utf8", mode: 0o700 }, ); await fs.writeFile( fixture.configPath, `${JSON.stringify( { secrets: { providers: { execmain: { source: "exec", command: execScriptPath, jsonOnly: true, passEnv: ["PATH"], }, }, }, models: { providers: { openai: { baseUrl: "https://api.openai.com/v1", api: "openai-completions", apiKey: { source: "exec", provider: "execmain", id: "providers/openai/apiKey" }, models: [{ id: "gpt-5", name: "gpt-5" }], }, moonshot: { baseUrl: "https://api.moonshot.cn/v1", api: "openai-completions", apiKey: { source: "exec", provider: "execmain", id: "providers/moonshot/apiKey" }, models: [{ id: "moonshot-v1-8k", name: "moonshot-v1-8k" }], }, }, }, }, null, 2, )}\n`, "utf8", ); await fs.rm(fixture.authStorePath, { force: true }); await fs.writeFile(fixture.envPath, "", "utf8"); const report = await runSecretsAudit({ env: fixture.env }); expect(report.summary.unresolvedRefCount).toBeGreaterThanOrEqual(2); const callLog = await fs.readFile(execLogPath, "utf8"); const callCount = callLog.split("\n").filter((line) => line.trim().length > 0).length; expect(callCount).toBe(1); }); it("scans agent models.json files for plaintext provider apiKey values", async () => { await writeModelsProvider({ apiKey: "sk-models-plaintext" }); // pragma: allowlist secret const report = await runSecretsAudit({ env: fixture.env }); expectModelsFinding(report, { code: "PLAINTEXT_FOUND", jsonPath: "providers.openai.apiKey", }); expect(report.filesScanned).toContain(fixture.modelsPath); }); it("scans agent models.json files for plaintext provider header values", async () => { await writeModelsProvider({ headers: { Authorization: "Bearer sk-header-plaintext", // pragma: allowlist secret }, }); const report = await runSecretsAudit({ env: fixture.env }); expectModelsFinding(report, { code: "PLAINTEXT_FOUND", jsonPath: "providers.openai.headers.Authorization", }); }); it("does not flag non-sensitive routing headers in models.json", async () => { await writeModelsProvider({ headers: { "X-Proxy-Region": "us-west", }, }); const report = await runSecretsAudit({ env: fixture.env }); expectModelsFinding(report, { code: "PLAINTEXT_FOUND", jsonPath: "providers.openai.headers.X-Proxy-Region", present: false, }); }); it("does not flag models.json marker values as plaintext", async () => { await writeModelsProvider(); const report = await runSecretsAudit({ env: fixture.env }); expectModelsFinding(report, { code: "PLAINTEXT_FOUND", jsonPath: "providers.openai.apiKey", present: false, }); }); it("flags arbitrary all-caps models.json apiKey values as plaintext", async () => { await writeModelsProvider({ apiKey: "ALLCAPS_SAMPLE" }); // pragma: allowlist secret const report = await runSecretsAudit({ env: fixture.env }); expectModelsFinding(report, { code: "PLAINTEXT_FOUND", jsonPath: "providers.openai.apiKey", }); }); it("does not flag models.json header marker values as plaintext", async () => { await writeModelsProvider({ headers: { Authorization: "secretref-env:OPENAI_HEADER_TOKEN", // pragma: allowlist secret "x-managed-token": "secretref-managed", // pragma: allowlist secret }, }); const report = await runSecretsAudit({ env: fixture.env }); expectModelsFinding(report, { code: "PLAINTEXT_FOUND", jsonPath: "providers.openai.headers.Authorization", present: false, }); expectModelsFinding(report, { code: "PLAINTEXT_FOUND", jsonPath: "providers.openai.headers.x-managed-token", present: false, }); }); it("reports unresolved models.json SecretRef objects in provider headers", async () => { await writeModelsProvider({ headers: { Authorization: { source: "env", provider: "default", id: "OPENAI_HEADER_TOKEN", // pragma: allowlist secret }, }, }); const report = await runSecretsAudit({ env: fixture.env }); expectModelsFinding(report, { code: "REF_UNRESOLVED", jsonPath: "providers.openai.headers.Authorization", }); }); it("reports malformed models.json as unresolved findings", async () => { await fs.writeFile(fixture.modelsPath, "{bad-json", "utf8"); const report = await runSecretsAudit({ env: fixture.env }); expectModelsFinding(report, { code: "REF_UNRESOLVED" }); }); it("reports non-regular models.json files as unresolved findings", async () => { await fs.rm(fixture.modelsPath, { force: true }); await fs.mkdir(fixture.modelsPath, { recursive: true }); const report = await runSecretsAudit({ env: fixture.env }); expectModelsFinding(report, { code: "REF_UNRESOLVED" }); }); it("reports oversized models.json as unresolved findings", async () => { const oversizedApiKey = "a".repeat(MAX_AUDIT_MODELS_JSON_BYTES + 256); await writeJsonFile(fixture.modelsPath, { providers: { openai: { baseUrl: "https://api.openai.com/v1", api: "openai-completions", apiKey: oversizedApiKey, models: [{ id: "gpt-5", name: "gpt-5" }], }, }, }); const report = await runSecretsAudit({ env: fixture.env }); expectModelsFinding(report, { code: "REF_UNRESOLVED" }); }); it("scans active agent-dir override models.json even when outside state dir", async () => { const externalAgentDir = path.join(fixture.rootDir, "external-agent"); const externalModelsPath = path.join(externalAgentDir, "models.json"); await fs.mkdir(externalAgentDir, { recursive: true }); await writeJsonFile(externalModelsPath, { providers: { openai: { baseUrl: "https://api.openai.com/v1", api: "openai-completions", apiKey: "sk-external-plaintext", // pragma: allowlist secret models: [{ id: "gpt-5", name: "gpt-5" }], }, }, }); const report = await runSecretsAudit({ env: { ...fixture.env, OPENCLAW_AGENT_DIR: externalAgentDir, }, }); expect( hasFinding( report, (entry) => entry.code === "PLAINTEXT_FOUND" && entry.file === externalModelsPath && entry.jsonPath === "providers.openai.apiKey", ), ).toBe(true); expect(report.filesScanned).toContain(externalModelsPath); }); it("does not flag non-sensitive routing headers in openclaw config", async () => { await writeJsonFile(fixture.configPath, { models: { providers: { openai: { baseUrl: "https://api.openai.com/v1", api: "openai-completions", apiKey: { source: "env", provider: "default", id: OPENAI_API_KEY_MARKER }, headers: { "X-Proxy-Region": "us-west", }, models: [{ id: "gpt-5", name: "gpt-5" }], }, }, }, }); await writeJsonFile(fixture.authStorePath, { version: 1, profiles: {}, }); await fs.writeFile(fixture.envPath, "", "utf8"); const report = await runSecretsAudit({ env: fixture.env }); expect( hasFinding( report, (entry) => entry.code === "PLAINTEXT_FOUND" && entry.file === fixture.configPath && entry.jsonPath === "models.providers.openai.headers.X-Proxy-Region", ), ).toBe(false); }); });