openclaw / src /cli /update-cli /update-command-executor.test.ts
SaylorTwift's picture
SaylorTwift HF Staff
Add files using upload-large-folder tool
eb3f11e verified
Raw
History Blame Contribute Delete
44 kB
import { spawn, spawnSync } from "node:child_process";
import { createHash, randomUUID } from "node:crypto";
import { once } from "node:events";
import fs from "node:fs";
import path from "node:path";
import { DatabaseSync } from "node:sqlite";
import { afterEach, assert, beforeEach, describe, expect, it, vi } from "vitest";
import { createDeferred } from "../../../test/helpers/promise.js";
import { useAutoCleanupTempDirTracker } from "../../../test/helpers/temp-dir.js";
import { resolveServiceManagerEnv } from "../../daemon/service-process-env.js";
import * as sqliteLocation from "../../infra/node-sqlite.js";
import { resolveRuntimeWorkerUrl } from "../../infra/runtime-worker-url.js";
import * as tempRoot from "../../infra/tmp-openclaw-dir.js";
import { CONTROL_PLANE_UPDATE_SENTINEL_META_ENV } from "../../infra/update-control-plane-sentinel.js";
import {
captureManagedUpdateLeaseDatabaseIdentity,
createManagedHandoffLeaseDatabase,
} from "../../infra/update-managed-service-handoff-database.js";
import { createManagedHandoffLeaseStore } from "../../infra/update-managed-service-handoff-lease.js";
import { MANAGED_HANDOFF_RUNTIME_ENTRY } from "../../infra/update-managed-service-handoff-runtime-assets.js";
import { stageManagedHandoffRuntime } from "../../infra/update-managed-service-handoff-runtime.js";
import { createUpdateRun, finishUpdateRun, getUpdateRun } from "../../infra/update-run-ledger.js";
import type { UpdateRecoveryFence } from "../../infra/update-run-recovery.js";
import { renderUpdateRunReport } from "../../infra/update-run-report.js";
import * as windowsProcess from "../../infra/windows-port-pids.js";
import { isChildProcessTreeAlive } from "../../process/child-process-tree.js";
import { runUtf8CommandWithTimeout } from "../../process/exec.js";
import * as pidAlive from "../../shared/pid-alive.js";
import { killPidIfAlive, waitForPidToExit } from "../../test-utils/process-tree.js";
import { withMockedPlatform } from "../../test-utils/vitest-spies.js";
import { updateExecutorNativeEntrypoints } from "./update-command-executor-native-runtime.test-support.js";
import {
captureUpdateCommandExecutorAuthority,
releaseUpdateCommandPreflightForHandoff,
withDelegatedUpdateCommandExecutor,
withUpdateCommandExecutor,
withUpdateCommandExecutorChild,
} from "./update-command-executor.js";
import { UpdateCommandRecoveryPendingError } from "./update-command-recovery.js";
const dirs = useAutoCleanupTempDirTracker(afterEach);
let root: string;
let temporary: string;
beforeEach(() => {
root = fs.realpathSync(dirs.make("update-executor-"));
temporary = path.join(root, "private-tmp");
fs.mkdirSync(temporary, { mode: 0o700 });
// Select only the private database location; the lease, process-start checks,
// and exact-row comparisons are the production owner.
vi.spyOn(tempRoot, "resolvePreferredOpenClawTmpDir").mockReturnValue(temporary);
});
afterEach(() => {
vi.restoreAllMocks();
vi.unstubAllEnvs();
});
// The installed parent prepares the database before a sealed actor can acquire a lease.
function prepareStagedLeaseFixture() {
const databasePath = path.join(temporary, "managed-update-handoffs.sqlite");
const existingIdentity = createManagedHandoffLeaseDatabase(databasePath)(true, () =>
captureManagedUpdateLeaseDatabaseIdentity(databasePath),
);
stageManagedHandoffRuntime(root);
return {
runtimeEntry: path.join(root, "runtime", MANAGED_HANDOFF_RUNTIME_ENTRY),
options: { databasePath, serviceManagerEnv: resolveServiceManagerEnv(), existingIdentity },
};
}
function replaceOwner(installationRoot = root) {
const db = new DatabaseSync(path.join(temporary, "managed-update-handoffs.sqlite"));
try {
db.prepare("UPDATE managed_update_handoffs SET owner = ? WHERE install_root = ?").run(
"replacement",
installationRoot,
);
} finally {
db.close();
}
}
describe("live update executor", () => {
it("finishes an attributed Windows candidate and records its missing start identity warning", async () => {
const hostPlatform = process.platform;
const existingUri = sqliteLocation.resolveExistingSqliteFileUri;
// Keep SQLite on the host VFS while exercising Windows process identity.
vi.spyOn(sqliteLocation, "resolveExistingSqliteFileUri").mockImplementation((pathname) =>
existingUri(pathname, hostPlatform),
);
const env = { HOME: root, OPENCLAW_STATE_DIR: root };
vi.stubEnv("OPENCLAW_STATE_DIR", root);
const run = createUpdateRun({ trigger: "cli" }, { env });
const candidatePid = 424242;
const argv = [
"C:\\node.exe",
"C:\\openclaw\\entry.js",
"gateway",
"install",
"--update-executor",
"check",
];
const readStart = pidAlive.getFileLockProcessStartTime;
const isDead = pidAlive.isPidDefinitelyDead;
let candidateAlive = true;
vi.spyOn(pidAlive, "getFileLockProcessStartTime").mockImplementation((pid, ...args) =>
pid === candidatePid ? null : readStart(pid, ...args),
);
vi.spyOn(pidAlive, "isPidDefinitelyDead").mockImplementation((pid) =>
pid === candidatePid ? !candidateAlive : isDead(pid),
);
vi.spyOn(windowsProcess, "readWindowsProcessArgsSync").mockReturnValue(argv);
const warning = vi.spyOn(console, "warn").mockImplementation(() => {});
await withUpdateCommandExecutor(run.runId, async (executor) => {
const fence = await executor.enter(root);
await withMockedPlatform("win32", () =>
withUpdateCommandExecutorChild(fence, root, async (_grant, bindChild) => {
try {
bindChild(candidatePid, argv);
} finally {
candidateAlive = false;
}
}),
);
fence.assertCurrent();
});
finishUpdateRun(run.runId, { status: "succeeded" }, { env });
const recorded = getUpdateRun(run.runId, { env });
assert(recorded);
expect(recorded.status).toBe("succeeded");
expect(recorded.steps).toContainEqual(
expect.objectContaining({
step: `warning:process-start-identity:${candidatePid}`,
status: "completed",
detail: expect.stringContaining("launcher attribution"),
}),
);
expect(renderUpdateRunReport(recorded).markdown).toContain("launcher attribution");
expect(warning).toHaveBeenCalledWith(expect.stringContaining(String(candidatePid)));
expect(createManagedHandoffLeaseStore().read(root)).toEqual({ kind: "absent" });
});
it("recovery acquires a fresh owner without reactivating the original fence", async () => {
const store = createManagedHandoffLeaseStore();
const runId = randomUUID();
const original = await withUpdateCommandExecutor(runId, async (executor) => {
const fence = await executor.enter(root);
const current = store.read(root);
assert(current.kind === "current", "Original executor was not acquired");
return {
fence,
lease: current.lease,
authority: captureUpdateCommandExecutorAuthority(fence),
};
});
expect(Object.isFrozen(original.authority)).toBe(true);
expect(original.authority.owner).toBe(original.lease.owner);
expect(() => captureUpdateCommandExecutorAuthority(original.fence)).toThrow(
"no longer current",
);
await withUpdateCommandExecutor(
runId,
async (executor) => {
const fence = await executor.enter(root);
const current = store.read(root);
assert(current.kind === "current", "Recovery executor was not acquired");
expect(current.lease.owner).not.toBe(original.lease.owner);
expect(current.lease.helper.pid).toBe(process.pid);
const recoveredAuthority = captureUpdateCommandExecutorAuthority(fence);
expect(recoveredAuthority).toEqual({
...original.authority,
owner: current.lease.owner,
});
expect(recoveredAuthority.owner).not.toBe(original.authority.owner);
expect(Object.isFrozen(recoveredAuthority)).toBe(true);
expect(store.current(original.lease)).toBe(false);
expect(store.release(original.lease)).toBe(false);
expect(original.fence.assertCurrent).toThrow("no longer current");
fence.assertCurrent();
},
{ existingAuthority: original.authority },
);
expect(store.read(root)).toEqual({ kind: "absent" });
});
it("recovery keeps the admitted installation key when the package root is missing", async () => {
const packageRoot = path.join(root, "package");
fs.mkdirSync(packageRoot);
const authority = await withUpdateCommandExecutor(randomUUID(), async (executor) =>
captureUpdateCommandExecutorAuthority(await executor.enter(packageRoot)),
);
fs.rmdirSync(packageRoot);
await withUpdateCommandExecutor(
randomUUID(),
async (executor) => {
const fence = await executor.enter(packageRoot);
fence.assertCurrent();
await expect(executor.enter(root)).rejects.toThrow("installation key changed");
const { owner: originalOwner, ...originalBinding } = authority;
const { owner: recoveredOwner, ...recoveredBinding } =
captureUpdateCommandExecutorAuthority(fence);
expect(recoveredBinding).toEqual(originalBinding);
expect(recoveredOwner).not.toBe(originalOwner);
expect(createManagedHandoffLeaseStore().read(packageRoot)).toMatchObject({
kind: "current",
lease: { owner: recoveredOwner },
});
},
{ existingAuthority: authority },
);
expect(fs.existsSync(packageRoot)).toBe(false);
expect(createManagedHandoffLeaseStore().read(packageRoot)).toEqual({ kind: "absent" });
});
it("retires the direct preflight owner before a supervised helper independently acquires", async () => {
const store = createManagedHandoffLeaseStore();
await withUpdateCommandExecutor(randomUUID(), async (executor) => {
const fence = await executor.enter(root, { preflight: true });
const original = store.read(root);
expect(original.kind).toBe("current");
releaseUpdateCommandPreflightForHandoff(fence);
expect(fence.assertCurrent).toThrow("no longer current");
expect(store.read(root)).toEqual({ kind: "absent" });
const acquired = store.acquire(root, "independent-helper", { kind: "update" });
expect(acquired.kind).toBe("acquired");
await expect(executor.enter(root)).rejects.toThrow("closed or busy");
expect(() => releaseUpdateCommandPreflightForHandoff(fence)).toThrow("not current");
if (acquired.kind === "acquired") {
expect(store.release(acquired.lease)).toBe(true);
}
});
});
it("closes preflight release on mutable admission without revoking its current owner", async () => {
await withUpdateCommandExecutor(randomUUID(), async (executor) => {
const fence = await executor.enter(root, { preflight: true });
expect(await executor.enter(root)).toBe(fence);
expect(() => releaseUpdateCommandPreflightForHandoff(fence)).toThrow("not current");
expect(fence.assertCurrent).not.toThrow();
});
});
it("refuses to release a replaced preflight owner and preserves the new lease", async () => {
await expect(
withUpdateCommandExecutor(randomUUID(), async (executor) => {
const fence = await executor.enter(root, { preflight: true });
replaceOwner();
expect(() => releaseUpdateCommandPreflightForHandoff(fence)).toThrow("no longer current");
const observed = createManagedHandoffLeaseStore().read(root);
expect(observed).toMatchObject({ kind: "current", lease: { owner: "replacement" } });
}),
).rejects.toThrow();
});
it("reclaims a dead direct executor through the existing process-liveness owner", async () => {
const { runtimeEntry, options } = prepareStagedLeaseFixture();
const result = spawnSync(
process.execPath,
[
"-e",
`
const {createManagedHandoffLeaseStore}=require(${JSON.stringify(runtimeEntry)});
const store=createManagedHandoffLeaseStore(${JSON.stringify(options)});
if(store.acquire(${JSON.stringify(root)},"dead-executor",{kind:"update"}).kind!=="acquired")throw new Error("admission failed");
`,
],
{ encoding: "utf8", timeout: 10_000 },
);
expect(result.status, result.stderr).toBe(0);
expect(createManagedHandoffLeaseStore().read(root).kind).toBe("current");
await withUpdateCommandExecutor(randomUUID(), async (executor) => {
const fence = await executor.enter(root);
fence.assertCurrent();
});
expect(createManagedHandoffLeaseStore().read(root)).toEqual({ kind: "absent" });
});
it("borrows only a live helper's exact assigned executor and leaves release to that helper", async () => {
const { runtimeEntry, options } = prepareStagedLeaseFixture();
const runId = randomUUID();
const owner = randomUUID();
const metadata = path.join(root, "handoff.json");
fs.writeFileSync(
metadata,
JSON.stringify({ version: 1, meta: { runId, handoffId: owner, root } }),
);
vi.stubEnv("OPENCLAW_UPDATE_RUN_HANDOFF", "1");
vi.stubEnv(CONTROL_PLANE_UPDATE_SENTINEL_META_ENV, metadata);
const child = spawn(
process.execPath,
[
"-e",
`
const {createManagedHandoffLeaseStore}=require(${JSON.stringify(runtimeEntry)});
const store=createManagedHandoffLeaseStore(${JSON.stringify(options)});
const acquired=store.acquire(${JSON.stringify(root)},${JSON.stringify(owner)},{kind:"update"});
if(acquired.kind!=="acquired")throw new Error("helper admission failed");
const assigned=store.bind(acquired.lease,${process.pid});
if(!assigned)throw new Error("helper assignment failed");
process.once("message",()=>{
const local=store.bind(assigned,process.pid);
if(!local||!store.release(local))throw new Error("helper release failed");
process.disconnect();
});
process.send("assigned");
`,
],
{ stdio: ["ignore", "ignore", "pipe", "ipc"] },
);
const exited = once(child, "exit");
let stderr = "";
child.stderr?.on("data", (data) => {
stderr += String(data);
});
try {
const ready = await Promise.race([
once(child, "message").then(([message]) => message),
exited.then(() => {
throw new Error(`helper exited before assignment: ${stderr}`);
}),
]);
expect(ready).toBe("assigned");
const store = createManagedHandoffLeaseStore();
await withUpdateCommandExecutor(runId, async (executor) => {
const fence = await executor.enter(root, { preflight: true });
expect(() => releaseUpdateCommandPreflightForHandoff(fence)).toThrow("not current");
await Promise.resolve();
fence.assertCurrent();
});
const current = store.read(root);
expect(current.kind === "current" && current.lease.owner).toBe(owner);
await expect(
withUpdateCommandExecutor(randomUUID(), async (executor) => executor.enter(root)),
).rejects.toThrow("changed during admission");
expect(store.read(root)).toEqual(current);
} finally {
if (child.connected) {
child.send("release");
}
const [code] = await exited;
expect(code, stderr).toBe(0);
}
expect(createManagedHandoffLeaseStore().read(root)).toEqual({ kind: "absent" });
});
it("preserves an unreadable existing coordination database without repairing it", async () => {
const database = path.join(temporary, "managed-update-handoffs.sqlite");
fs.writeFileSync(database, "unreadable native owner", { mode: 0o600 });
const before = fs.readFileSync(database);
await expect(
withUpdateCommandExecutor(randomUUID(), async (executor) => executor.enter(root)),
).rejects.toThrow("state is unreadable");
expect(fs.readFileSync(database)).toEqual(before);
});
it("does not open the coordination database for a read-only or no-op invocation", async () => {
await withUpdateCommandExecutor(randomUUID(), async () => "preview");
expect(fs.readdirSync(temporary)).toEqual([]);
});
it("holds the existing owner across awaited execution and refuses a second local invocation", async () => {
const admitted = createDeferred();
const settle = createDeferred();
const runId = randomUUID();
let retained: UpdateRecoveryFence | undefined;
const running = withUpdateCommandExecutor(runId, async (executor) => {
retained = await executor.enter(root);
retained.assertCurrent();
admitted.resolve();
await settle.promise;
retained.assertCurrent();
return "completed";
});
try {
await admitted.promise;
const store = createManagedHandoffLeaseStore();
expect(store.read(root).kind).toBe("current");
await expect(
withUpdateCommandExecutor(runId, async (other) => other.enter(root)),
).rejects.toThrow("Another update executor");
retained!.assertCurrent();
} finally {
settle.resolve();
}
await expect(running).resolves.toBe("completed");
expect(createManagedHandoffLeaseStore().read(root)).toEqual({ kind: "absent" });
expect(() => retained!.assertCurrent()).toThrow("no longer current");
});
it("rejects a changed native-owner row after an await without removing the replacement", async () => {
await expect(
withUpdateCommandExecutor(randomUUID(), async (executor) => {
const fence = await executor.enter(root);
await Promise.resolve();
replaceOwner();
fence.assertCurrent();
}),
).rejects.toBeInstanceOf(UpdateCommandRecoveryPendingError);
const current = createManagedHandoffLeaseStore().read(root);
expect(current.kind === "current" && current.lease.owner).toBe("replacement");
});
it("preserves the primary error and releases only its own exact owner", async () => {
const primary = new Error("package failed");
await expect(
withUpdateCommandExecutor(randomUUID(), async (executor) => {
await executor.enter(root);
throw primary;
}),
).rejects.toBe(primary);
expect(createManagedHandoffLeaseStore().read(root)).toEqual({ kind: "absent" });
});
it("closes saved admission methods without minting a later owner", async () => {
const saved = await withUpdateCommandExecutor(randomUUID(), async (executor) => executor);
await expect(saved.enter(root)).rejects.toThrow("admission is closed");
expect(fs.readdirSync(temporary)).toEqual([]);
});
it("pins the originally admitted installation for the full invocation", async () => {
const moved = path.join(root, "different-install");
fs.mkdirSync(moved);
await withUpdateCommandExecutor(randomUUID(), async (executor) => {
const fence = await executor.enter(root);
await expect(executor.enter(moved)).rejects.toThrow("installation changed");
fence.assertCurrent();
});
expect(createManagedHandoffLeaseStore().read(moved)).toEqual({ kind: "absent" });
});
});
describe("candidate executor delegation", () => {
const moduleUrl = new URL("./update-command-executor.ts", import.meta.url).href;
it.each([
{ mismatched: false, becomesReadable: false, revoked: false },
{ mismatched: true, becomesReadable: false, revoked: false },
{ mismatched: false, becomesReadable: true, revoked: false },
{ mismatched: false, becomesReadable: false, revoked: true },
])(
"consumes the parent's bound creation identity when the Windows receiver cannot read its own (mismatch=$mismatched, fallback becomes readable=$becomesReadable, revoked=$revoked)",
async ({ mismatched, becomesReadable, revoked }) => {
const hostPlatform = process.platform;
const existingUri = sqliteLocation.resolveExistingSqliteFileUri;
vi.spyOn(sqliteLocation, "resolveExistingSqliteFileUri").mockImplementation((pathname) =>
existingUri(pathname, hostPlatform),
);
const readStart = pidAlive.getFileLockProcessStartTime;
const parentStart = readStart(process.ppid);
const receiverStart = readStart(process.pid);
assert(parentStart !== null);
assert(receiverStart !== null);
const store = createManagedHandoffLeaseStore();
const runId = randomUUID();
const childKey = `${root}/.openclaw-update-child-${randomUUID()}`;
assert(store.acquire(root, randomUUID(), { kind: "update" }).kind === "acquired");
assert(store.acquire(childKey, runId, { kind: "update" }).kind === "acquired");
const platform = vi.spyOn(process, "platform", "get").mockReturnValue("win32");
let currentReceiverStart = mismatched ? receiverStart + 1 : null;
vi.spyOn(pidAlive, "getFileLockProcessStartTime").mockImplementation((pid, ...args) => {
if (pid === process.pid) {
return currentReceiverStart;
}
if (pid === process.ppid) {
return parentStart;
}
platform.mockReturnValue(hostPlatform);
try {
return readStart(pid, ...args);
} finally {
platform.mockReturnValue("win32");
}
});
const parentIdentity = { pid: process.ppid, startIdentity: String(parentStart) };
const receiverIdentity =
becomesReadable || revoked
? createManagedHandoffLeaseStore().processIdentity()
: { pid: process.pid, startIdentity: String(receiverStart) };
const databasePath = path.join(temporary, "managed-update-handoffs.sqlite");
const db = new DatabaseSync(databasePath);
try {
// Reproduce the rows already bound by the parent before releasing private stdin.
const update = db.prepare(
"UPDATE managed_update_handoffs SET payload_json = ? WHERE install_root = ?",
);
for (const [key, executor] of [
[root, parentIdentity],
[childKey, receiverIdentity],
] as const) {
update.run(
JSON.stringify({
version: 2,
helper: parentIdentity,
executor,
action: { kind: "update" },
}),
key,
);
}
} finally {
db.close();
}
const parent = store.read(root);
assert(parent.kind === "current");
const warning = vi.spyOn(console, "warn").mockImplementation(() => {});
const effectPath = path.join(root, "authorized-effect");
let nestedStarted = false;
const operation = vi.fn(async (fence: UpdateRecoveryFence) => {
fence.assertCurrent();
if (becomesReadable) {
currentReceiverStart = receiverStart;
}
if (revoked) {
replaceOwner();
}
let nestedKey: string | undefined;
await withUpdateCommandExecutorChild(fence, root, async (grant, bindChild) => {
nestedStarted = true;
nestedKey = grant.childKey;
const candidate = spawn(process.execPath, ["-e", "process.stdin.resume()"], {
stdio: ["pipe", "ignore", "ignore"],
});
const exited = once(candidate, "exit");
try {
await once(candidate, "spawn");
assert(candidate.pid);
bindChild(candidate.pid, candidate.spawnargs);
const nested = store.read(grant.childKey);
assert(nested.kind === "current");
expect(nested.lease.helper).toEqual(receiverIdentity);
candidate.stdin.end();
await exited;
} finally {
if (candidate.exitCode === null && candidate.signalCode === null) {
candidate.kill("SIGKILL");
}
await exited;
}
});
assert(nestedKey);
expect(store.read(nestedKey)).toEqual({ kind: "absent" });
fence.assertCurrent();
fs.writeFileSync(effectPath, "owned");
return "completed";
});
const result = withDelegatedUpdateCommandExecutor(
{ runId, root, databasePath, parent: parent.lease, childKey },
runId,
root,
operation,
);
if (mismatched) {
await expect(result).rejects.toThrow(/ownership|identity/);
} else if (revoked) {
await expect(result).rejects.toThrow(/ownership|settlement/);
} else {
await expect(result).resolves.toBe("completed");
}
expect(operation).toHaveBeenCalledTimes(mismatched ? 0 : 1);
expect(nestedStarted).toBe(!mismatched && !revoked);
expect(fs.existsSync(effectPath)).toBe(!mismatched && !revoked);
if (!mismatched && !becomesReadable && !revoked) {
expect(warning).toHaveBeenCalledWith(
expect.stringContaining("established by the live parent"),
);
}
},
);
it("refuses a revoked requester before delegated Doctor changes operator config", async () => {
const configPath = path.join(root, "openclaw.json");
const original = JSON.stringify({
commands: { ownerAllowFrom: ["replacement"] },
plugins: { enabled: false },
});
fs.writeFileSync(configPath, original);
const workerUrl = resolveRuntimeWorkerUrl(updateExecutorNativeEntrypoints.migratedFinalize);
const resultUrl = resolveRuntimeWorkerUrl(updateExecutorNativeEntrypoints.doctorResult);
// Prepare the worker before its deadline; watch mode still loads live source.
const sourceImportArgs = workerUrl.pathname.endsWith(".ts")
? ["--import", path.resolve("scripts/tsx.mjs")]
: [];
const childProgram = `
import fs from "node:fs";
import {createUpdatePostInstallDoctorResultPath, UPDATE_POST_INSTALL_DOCTOR_RESULT_PATH_ENV} from ${JSON.stringify(resultUrl.href)};
const resultPath = createUpdatePostInstallDoctorResultPath();
process.env[UPDATE_POST_INSTALL_DOCTOR_RESULT_PATH_ENV] = resultPath;
process.argv[2] = "--doctor";
process.once("exit", () => {
if (fs.existsSync(resultPath)) {
process.stdout.write(fs.readFileSync(resultPath, "utf8"));
fs.rmSync(resultPath);
}
});
await import(${JSON.stringify(workerUrl.href)});
`;
const runId = randomUUID();
await withUpdateCommandExecutor(runId, async (executor) => {
const fence = await executor.enter(root);
const result = await withUpdateCommandExecutorChild(fence, root, (grant, beforeInput) =>
runUtf8CommandWithTimeout(
[process.execPath, ...sourceImportArgs, "--input-type=module", "-e", childProgram],
{
input: JSON.stringify({
executor: grant,
runId,
root,
configInputHash: createHash("sha256").update(original).digest("hex"),
requester: { channel: "synthetic", senderId: "owner" },
repair: true,
}),
beforeInput,
env: { HOME: root, OPENCLAW_STATE_DIR: root, OPENCLAW_CONFIG_PATH: configPath },
timeoutMs: 15_000,
killProcessTree: true,
requireProcessTreeExtinction: true,
},
),
);
expect(result.code, result.stderr).toBe(1);
expect(result.stdout, result.stderr).toContain('"reason":"requester-revoked"');
expect(JSON.parse(result.stdout)).toMatchObject({
status: "error",
configWriteRefusal: { reason: "requester-revoked", keys: [] },
});
expect(fs.readFileSync(configPath, "utf8")).toBe(original);
fence.assertCurrent();
});
});
const program = `
import fs from "node:fs";
import {spawn} from "node:child_process";
import {once} from "node:events";
import {setTimeout} from "node:timers/promises";
import {withDelegatedUpdateCommandExecutor} from ${JSON.stringify(moduleUrl)};
const input=JSON.parse(fs.readFileSync(0,"utf8"));
await withDelegatedUpdateCommandExecutor(input.grant,input.grant.runId,input.root,async (fence)=>{
process.stdout.write("admitted\\n");
while(!fs.existsSync(input.proceed)) await setTimeout(10);
fence.assertCurrent();
fs.writeFileSync(input.output,"owned");
const helper=spawn(process.execPath,['-e',"process.send('ready');setTimeout(()=>{},2000)"],{
stdio:['ignore','ignore','ignore','ipc']
});
await once(helper,'message');
helper.disconnect();
helper.unref();
});
`;
it.each([
{ changedRoot: false, revoked: false },
{ changedRoot: false, revoked: "candidate" },
{ changedRoot: true, revoked: false },
{ changedRoot: true, revoked: "candidate" },
{ changedRoot: true, revoked: "original" },
])(
"retains both installation owners through a real child ($changedRoot, $revoked)",
async ({ changedRoot, revoked }) => {
const candidateRoot = changedRoot ? path.join(root, "activated") : root;
if (changedRoot) {
fs.mkdirSync(candidateRoot);
}
const ready = createDeferred();
const proceed = path.join(root, "proceed");
const output = path.join(root, "effect");
const work = withUpdateCommandExecutor(randomUUID(), async (executor) => {
const fence = await executor.enter(root);
const pending = withUpdateCommandExecutorChild(fence, candidateRoot, (grant, beforeInput) =>
runUtf8CommandWithTimeout(
[
process.execPath,
"--import",
path.resolve("scripts/tsx.mjs"),
"--input-type=module",
"-e",
program,
],
{
input: JSON.stringify({ grant, root: candidateRoot, proceed, output }),
beforeInput,
timeoutMs: 15_000,
killProcessTree: true,
// Match production candidate transport: join source-loader helpers too.
requireProcessTreeExtinction: true,
onOutputChunk: (chunk) => {
if (chunk.toString().includes("admitted")) {
ready.resolve();
}
},
},
),
);
try {
await Promise.race([
ready.promise,
pending.then((result) => {
throw new Error(result.stderr);
}),
]);
expect(() => fence.assertCurrent()).toThrow("suspended");
const store = createManagedHandoffLeaseStore();
const primary = store.read(root);
expect(primary.kind).toBe("current");
if (primary.kind !== "current") {
throw new Error("missing primary owner");
}
expect(store.release(primary.lease)).toBe(false);
expect(store.bind(primary.lease, process.pid)).toBeNull();
expect(store.acquire(candidateRoot, "other-candidate", { kind: "update" }).kind).toBe(
"busy",
);
if (revoked) {
replaceOwner(revoked === "original" ? root : candidateRoot);
}
} finally {
fs.writeFileSync(proceed, "continue");
}
const result = await pending;
expect(result.code, result.stderr).toBe(0);
fence.assertCurrent();
});
if (revoked) {
await expect(work).rejects.toThrow(/ownership|release/);
// The current receiver also retains the original recovery owner after
// activation: changing either owner must refuse the child effect.
expect(fs.existsSync(output)).toBe(false);
expect(
createManagedHandoffLeaseStore().read(revoked === "original" ? root : candidateRoot),
).toMatchObject({
kind: "current",
lease: { owner: "replacement" },
});
if (changedRoot && revoked === "candidate") {
expect(
createManagedHandoffLeaseStore().acquire(root, "next-original", { kind: "update" })
.kind,
).toBe("busy");
}
if (revoked === "original") {
expect(createManagedHandoffLeaseStore().read(candidateRoot)).toEqual({ kind: "absent" });
}
} else {
await work;
expect(fs.readFileSync(output, "utf8")).toBe("owned");
expect(createManagedHandoffLeaseStore().read(root)).toEqual({ kind: "absent" });
expect(createManagedHandoffLeaseStore().read(candidateRoot)).toEqual({ kind: "absent" });
}
},
);
it.skipIf(process.platform === "win32").each([false, true])(
"does not release either installation while a candidate descendant is alive (changed root=%s)",
async (changedRoot) => {
const candidateRoot = changedRoot ? path.join(root, "activated") : root;
if (changedRoot) {
fs.mkdirSync(candidateRoot);
}
let descendant: number | undefined;
try {
await expect(
withUpdateCommandExecutor(randomUUID(), async (executor) => {
const fence = await executor.enter(root);
await withUpdateCommandExecutorChild(
fence,
candidateRoot,
async (grant, beforeInput) => {
const result = await runUtf8CommandWithTimeout(
[
process.execPath,
"-e",
`const fs=require('node:fs');const {spawn}=require('node:child_process');
JSON.parse(fs.readFileSync(0,'utf8'));
const child=spawn(process.execPath,['-e',"setInterval(()=>{},1000);process.send('ready')"],{stdio:['ignore','ignore','ignore','ipc']});
child.once('message',()=>{process.stdout.write(String(child.pid));child.disconnect();child.unref();});`,
],
{
input: JSON.stringify(grant),
beforeInput,
killProcessTree: true,
timeoutMs: 15_000,
},
);
descendant = Number(result.stdout);
expect(result.code, result.stderr).toBe(0);
expect(Number.isSafeInteger(descendant) && descendant > 0).toBe(true);
process.kill(descendant, 0);
return result;
},
);
}),
).rejects.toThrow(/settled|release/);
const store = createManagedHandoffLeaseStore();
expect(store.acquire(root, "next-owner", { kind: "update" }).kind).toBe("busy");
expect(store.acquire(candidateRoot, "next-candidate", { kind: "update" }).kind).toBe(
"busy",
);
} finally {
if (descendant) {
killPidIfAlive(descendant);
expect(await waitForPidToExit(descendant)).toBe(true);
}
}
},
);
it("does not expose a grant when another owner holds the activated installation", async () => {
const candidateRoot = path.join(root, "activated");
const output = path.join(root, "exposed-grant");
fs.mkdirSync(candidateRoot);
const store = createManagedHandoffLeaseStore();
const foreign = store.acquire(candidateRoot, "foreign-candidate", { kind: "update" });
assert(foreign.kind === "acquired", "Foreign candidate owner was not acquired");
try {
await expect(
withUpdateCommandExecutor(randomUUID(), async (executor) => {
const fence = await executor.enter(root);
await withUpdateCommandExecutorChild(fence, candidateRoot, async () => {
fs.writeFileSync(output, "exposed");
});
}),
).rejects.toThrow("owns the candidate installation");
expect(fs.existsSync(output)).toBe(false);
expect(store.current(foreign.lease)).toBe(true);
expect(store.read(root)).toEqual({ kind: "absent" });
} finally {
expect(store.release(foreign.lease)).toBe(true);
}
});
it("withholds candidate input when the original owner changes before process binding", async () => {
const candidateRoot = path.join(root, "activated");
const output = path.join(root, "effect");
fs.mkdirSync(candidateRoot);
await expect(
withUpdateCommandExecutor(randomUUID(), async (executor) => {
const fence = await executor.enter(root);
await withUpdateCommandExecutorChild(fence, candidateRoot, (grant, beforeInput) =>
runUtf8CommandWithTimeout(
[
process.execPath,
"--import",
path.resolve("scripts/tsx.mjs"),
"--input-type=module",
"-e",
program,
],
{
input: JSON.stringify({ grant, root: candidateRoot, output }),
beforeInput: (pid) => {
replaceOwner();
beforeInput(pid);
},
timeoutMs: 15_000,
killProcessTree: true,
requireProcessTreeExtinction: true,
},
),
);
}),
).rejects.toThrow(/ownership|release/);
expect(fs.existsSync(output)).toBe(false);
expect(createManagedHandoffLeaseStore().read(root)).toMatchObject({
kind: "current",
lease: { owner: "replacement" },
});
expect(createManagedHandoffLeaseStore().read(candidateRoot)).toEqual({ kind: "absent" });
});
it.each([false, true])(
"rejects a changed parent grant and settles its child (changed root=%s)",
async (changedRoot) => {
const candidateRoot = changedRoot ? path.join(root, "activated") : root;
if (changedRoot) {
fs.mkdirSync(candidateRoot);
}
const output = path.join(root, "effect");
await withUpdateCommandExecutor(randomUUID(), async (executor) => {
const fence = await executor.enter(root);
const result = await withUpdateCommandExecutorChild(
fence,
candidateRoot,
(grant, beforeInput) =>
runUtf8CommandWithTimeout(
[
process.execPath,
"--import",
path.resolve("scripts/tsx.mjs"),
"--input-type=module",
"-e",
program,
],
{
input: JSON.stringify({
grant: {
...grant,
parent: { ...grant.parent, updatedAt: grant.parent.updatedAt + 1 },
},
output,
root: candidateRoot,
}),
beforeInput,
timeoutMs: 15_000,
killProcessTree: true,
},
),
);
expect(result.code).not.toBe(0);
expect(result.stderr).toContain("does not match its parent");
expect(fs.existsSync(output)).toBe(false);
fence.assertCurrent();
});
},
);
it.skipIf(process.platform === "win32")(
"retains a candidate group after both the updater and its direct child exit",
async () => {
const { runtimeEntry, options } = prepareStagedLeaseFixture();
const command = `
const {spawn}=require('node:child_process');
process.stdin.once('data',()=>{
const leaf=spawn(process.execPath,['-e',"setInterval(()=>{},1000);process.send('ready')"],{stdio:['ignore','ignore','ignore','ipc']});
leaf.once('message',()=>{process.stdout.write(String(leaf.pid));leaf.disconnect();leaf.unref();});
});
`;
const parent = spawnSync(
process.execPath,
[
"-e",
`
const {spawn}=require('node:child_process');
const {createManagedHandoffLeaseStore}=require(${JSON.stringify(runtimeEntry)});
const store=createManagedHandoffLeaseStore(${JSON.stringify(options)});
const original=store.acquire(${JSON.stringify(root)},'parent',{kind:'update'});
const delegation=store.acquire(${JSON.stringify(root + "/.openclaw-update-child-group")},'run',{kind:'update'});
if(original.kind!=='acquired'||delegation.kind!=='acquired')throw new Error('admission failed');
const child=spawn(process.execPath,['-e',${JSON.stringify(command)}],{detached:true,stdio:['pipe','pipe','inherit']});
if(!store.bind(delegation.lease,child.pid))throw new Error('bind failed');
child.stdout.pipe(process.stdout);child.stdin.end('start');
`,
],
{ encoding: "utf8", timeout: 15_000 },
);
expect(parent.status, parent.stderr).toBe(0);
const descendant = Number(parent.stdout);
expect(Number.isSafeInteger(descendant) && descendant > 0).toBe(true);
const store = createManagedHandoffLeaseStore();
try {
expect(store.acquire(root, "new", { kind: "update" }).kind).toBe("busy");
} finally {
process.kill(descendant, "SIGTERM");
const candidate = store.read(root + "/.openclaw-update-child-group");
assert(candidate.kind === "current", "Candidate group lease is missing");
// A Linux zombie has exited but retains its process group until reaped.
await vi.waitFor(
() => expect(isChildProcessTreeAlive(candidate.lease.executor)).toBe(false),
{ timeout: 2_000, interval: 25 },
);
}
const next = store.acquire(root, "new", { kind: "update" });
expect(next.kind).toBe("acquired");
if (next.kind === "acquired") {
expect(store.release(next.lease)).toBe(true);
}
},
);
it("does not reclaim a dead parent while its delegated child is alive", async () => {
const { runtimeEntry, options } = prepareStagedLeaseFixture();
const parent = spawnSync(
process.execPath,
[
"-e",
`
const {spawn}=require("node:child_process");
const {createManagedHandoffLeaseStore}=require(${JSON.stringify(runtimeEntry)});
const store=createManagedHandoffLeaseStore(${JSON.stringify(options)});
const parent=store.acquire(${JSON.stringify(root)},"parent",{kind:"update"});
const delegated=store.acquire(${JSON.stringify(root + "/.openclaw-update-child-test")},"run",{kind:"update"});
if(parent.kind!=="acquired"||delegated.kind!=="acquired")throw new Error("admission failed");
const child=spawn(process.execPath,["-e","setInterval(()=>{},1000)"],{stdio:"ignore",detached:true});
child.unref();
if(!store.bind(delegated.lease,child.pid))throw new Error("bind failed");
process.stdout.write(String(child.pid));
`,
],
{ encoding: "utf8", timeout: 15_000 },
);
expect(parent.status, parent.stderr).toBe(0);
const pid = Number(parent.stdout);
expect(Number.isInteger(pid) && pid > 0).toBe(true);
const existingAuthority = {
...captureManagedUpdateLeaseDatabaseIdentity(options.databasePath),
installKey: root,
};
const recover = () =>
withUpdateCommandExecutor(
randomUUID(),
async (executor) => {
const fence = await executor.enter(root);
fence.assertCurrent();
const recovered = createManagedHandoffLeaseStore().read(root);
assert(recovered.kind === "current", "Recovery executor was not acquired");
expect(recovered.lease.owner).not.toBe("parent");
},
{ existingAuthority },
);
try {
expect(createManagedHandoffLeaseStore().acquire(root, "new", { kind: "update" }).kind).toBe(
"busy",
);
await expect(recover()).rejects.toThrow("Another update executor");
} finally {
process.kill(pid, "SIGTERM");
await vi.waitFor(() => expect(isChildProcessTreeAlive({ pid })).toBe(false), {
timeout: 2_000,
interval: 25,
});
}
await recover();
const store = createManagedHandoffLeaseStore();
const acquired = store.acquire(root, "new", { kind: "update" });
expect(acquired.kind).toBe("acquired");
if (acquired.kind === "acquired") {
expect(store.release(acquired.lease)).toBe(true);
}
});
});