// Gateway service lifecycle command core: install, uninstall, start, stop, restart. import type { Writable } from "node:stream"; import { readBestEffortConfig } from "../../config/config.js"; import { resolveIsNixMode } from "../../config/paths.js"; import { checkTokenDrift } from "../../daemon/service-audit.js"; import type { GatewayServiceRestartResult } from "../../daemon/service-types.js"; import { assertGatewayServiceUpdateCurrent } from "../../daemon/service-update-authority.js"; import type { GatewayServiceStartRepairIssue, GatewayServiceState, GatewayService, } from "../../daemon/service.js"; import { describeGatewayServiceRestart, inspectGatewayServiceStartRepair, readGatewayServiceLoadState, startGatewayService, } from "../../daemon/service.js"; import { renderSystemdUnavailableHints } from "../../daemon/systemd-hints.js"; import { isSystemdUserServiceAvailable } from "../../daemon/systemd.js"; import { isGatewaySecretRefUnavailableError } from "../../gateway/credentials.js"; import { clearGatewayRestartIntentSync, type GatewayRestartIntent, writeGatewayRestartIntentSync, } from "../../infra/restart-intent.js"; import { isWSL } from "../../infra/wsl.js"; import { defaultRuntime } from "../../runtime.js"; import { formatCliCommand } from "../command-format.js"; import { formatInvalidConfigRecoveryHint } from "../config-recovery-hints.js"; import { resolveGatewayTokenForDriftCheck } from "./gateway-token-drift.js"; import { getServiceActionPreflightFailure } from "./lifecycle-action-preflight.js"; import { appendServiceLifecycleRepairAudit, createServiceLifecycleMutationAudit, } from "./lifecycle-audit.js"; import { buildDaemonServiceSnapshot, createDaemonActionContext, emitDaemonAlreadyRunning, emitDaemonScheduledRestart, } from "./response.js"; import { filterContainerGenericHints, resolveDaemonInstallBlockMessage } from "./shared.js"; type DaemonLifecycleOptions = { json?: boolean; force?: boolean; wait?: string; restartIntent?: GatewayRestartIntent; preserveDefinition?: boolean; disable?: boolean; }; type StartPostCheckContext = { json: boolean; stdout: Writable; warnings: string[]; warn?: (message: string) => void; fail: ReturnType["fail"]; }; type RestartPostCheckContext = StartPostCheckContext & { activationAccepted: boolean }; type ServiceRecoveryResult = { result: TResult; message?: string; warnings?: string[]; loaded?: boolean; }; type ServiceRecoveryContext = { json: boolean; stdout: Writable; warn?: (message: string) => void; fail: (message: string, hints?: string[]) => void; }; type ServiceStartRepairContext = ServiceRecoveryContext & { state: GatewayServiceState; issues: GatewayServiceStartRepairIssue[]; }; async function maybeAugmentSystemdHints(hints: string[]): Promise { if (process.platform !== "linux") { return hints; } const systemdAvailable = await isSystemdUserServiceAvailable().catch(() => false); if (systemdAvailable) { return hints; } return [ ...hints, ...renderSystemdUnavailableHints({ wsl: await isWSL(), kind: "generic_unavailable" }), ]; } function mergeWarnings( captured: readonly string[], reported?: readonly string[], ): string[] | undefined { const combined = [...captured, ...(reported ?? [])]; return combined.length > 0 ? combined : undefined; } async function failServiceNotLoaded(params: { serviceNoun: string; service: GatewayService; renderStartHints: () => string[]; fail: ReturnType["fail"]; }) { const hints = filterContainerGenericHints( await maybeAugmentSystemdHints(params.renderStartHints()), ); params.fail(`${params.serviceNoun} service ${params.service.notLoadedText}.`, hints); } async function resolveServiceLoadedOrFail(params: { serviceNoun: string; service: GatewayService; fail: ReturnType["fail"]; acceptInstalledDefinition?: boolean; inspectionFailureMessage?: string; }): Promise { // Keep native scope discovery in the adapter and failure emission in the action context. const hasInstalledDefinition = async () => params.service.hasInstalledDefinition ? await params.service.hasInstalledDefinition({ env: process.env }).catch(() => false) : Boolean(await params.service.readCommand(process.env).catch(() => null)); const loadState = await readGatewayServiceLoadState(params.service, { env: process.env }); if (loadState.status === "unknown") { params.fail( `${params.inspectionFailureMessage ?? `${params.serviceNoun} service check failed`}: ${loadState.detail}`, ); return null; } return ( loadState.status === "loaded" || (Boolean(params.acceptInstalledDefinition) && (await hasInstalledDefinition())) ); } export async function runServiceUninstall(params: { serviceNoun: string; service: GatewayService; opts?: DaemonLifecycleOptions; stopBeforeUninstall: boolean; assertNotLoadedAfterUninstall: boolean; }) { const json = Boolean(params.opts?.json); const { stdout, emit, fail } = createDaemonActionContext({ action: "uninstall", json }); if (resolveIsNixMode(process.env)) { fail("Nix mode detected; service uninstall is disabled."); return; } { const preflight = await getServiceActionPreflightFailure("uninstall"); if (preflight) { fail(`${params.serviceNoun} uninstall blocked: ${preflight.message}`, preflight.hints); return; } } let loaded = await resolveServiceLoadedOrFail({ serviceNoun: params.serviceNoun, service: params.service, fail, inspectionFailureMessage: `${params.serviceNoun} uninstall aborted because service status is unknown; resolve the inspection error before retrying`, }); if (loaded === null) { return; } if (loaded && params.stopBeforeUninstall) { try { await params.service.stop({ env: process.env, stdout }); } catch { // Best-effort stop; final loaded check gates success when enabled. } } try { await params.service.uninstall({ env: process.env, stdout }); } catch (err) { fail(`${params.serviceNoun} uninstall failed: ${String(err)}`); return; } loaded = await resolveServiceLoadedOrFail({ serviceNoun: params.serviceNoun, service: params.service, fail, inspectionFailureMessage: `${params.serviceNoun} uninstall verification failed because service status is unknown`, }); if (loaded === null) { return; } if (loaded && params.assertNotLoadedAfterUninstall) { fail(`${params.serviceNoun} service still loaded after uninstall.`); return; } emit({ ok: true, result: "uninstalled", service: buildDaemonServiceSnapshot(params.service, loaded), }); } export async function runServiceStart(params: { serviceNoun: string; service: GatewayService; renderStartHints: () => string[]; opts?: DaemonLifecycleOptions; onNotLoaded?: (ctx: ServiceRecoveryContext) => Promise | null>; repairLoadedService?: ( ctx: ServiceStartRepairContext, ) => Promise | null>; /** Runs after the service process is started, before start reports success. */ postStartCheck?: (ctx: StartPostCheckContext) => Promise; expectedPort?: number; }) { const json = Boolean(params.opts?.json); const serviceCommand = formatCliCommand(`openclaw ${params.serviceNoun.toLowerCase()}`); const { stdout, warnings, emit, fail } = createDaemonActionContext({ action: "start", json }); const warn = json ? (message: string) => warnings.push(message) : undefined; const emitStarted = async (result: { loaded: boolean; message?: string; reportedWarnings?: readonly string[]; }) => { await params.postStartCheck?.({ json, stdout, warnings, warn, fail }); emit({ ok: true, result: "started", message: result.message, warnings: mergeWarnings(warnings, result.reportedWarnings), service: buildDaemonServiceSnapshot(params.service, result.loaded), }); if (!json && result.message) { defaultRuntime.log(result.message); } }; const loaded = await resolveServiceLoadedOrFail({ serviceNoun: params.serviceNoun, service: params.service, fail, }); if (loaded === null) { return; } // Pre-flight config validation (#35862) — run for both loaded and not-loaded // to prevent launching from invalid config in any start path. { const preflight = await getServiceActionPreflightFailure("start"); if (preflight) { fail( preflight.hints ? `${params.serviceNoun} start blocked: ${preflight.message}` : `${params.serviceNoun} aborted: config is invalid.\n${preflight.message}\n${formatInvalidConfigRecoveryHint()}`, preflight.hints, ); return; } } if (!loaded) { try { const handled = await params.onNotLoaded?.({ json, stdout, warn, fail }); if (handled) { await emitStarted({ loaded: handled.loaded ?? false, message: handled.message, reportedWarnings: handled.warnings, }); return; } } catch (err) { fail(`${params.serviceNoun} start failed: ${String(err)}`, params.renderStartHints()); return; } } try { const startResult = await startGatewayService( params.service, { env: process.env, stdout, warn, onMutation: createServiceLifecycleMutationAudit({ serviceNoun: params.serviceNoun, action: "start", }), }, params.expectedPort, ); if (startResult.outcome === "missing-install") { await failServiceNotLoaded({ serviceNoun: params.serviceNoun, service: params.service, renderStartHints: params.renderStartHints, fail, }); return; } if (startResult.outcome === "already-running") { if (startResult.issues.length > 0) { // Only services with a repair callback can rebuild their definition during restart. const repairAction = params.repairLoadedService ? "restart" : "install --force"; const warning = `${params.serviceNoun} service already running, but its installed service definition needs repair: ${startResult.issues .map((issue) => issue.message) .join("; ")}; run \`${serviceCommand} ${repairAction}\` to apply.`; warnings.push(warning); if (!json) { defaultRuntime.log(warning); } } emitDaemonAlreadyRunning({ serviceNoun: params.serviceNoun, service: params.service, pid: startResult.state.runtime?.pid, json, warnings, emit, }); return; } if (startResult.outcome === "repair-required") { try { const handled = await params.repairLoadedService?.({ json, stdout, warn, fail, state: startResult.state, issues: startResult.issues, }); if (handled) { appendServiceLifecycleRepairAudit({ serviceNoun: params.serviceNoun, action: "start", }); await emitStarted({ loaded: handled.loaded ?? true, message: handled.message, reportedWarnings: handled.warnings, }); return; } } catch (err) { fail(`${params.serviceNoun} repair failed: ${String(err)}`, params.renderStartHints()); return; } fail( `${params.serviceNoun} service needs repair before it can start: ${startResult.issues .map((issue) => issue.message) .join("; ")}`, [`${serviceCommand} install --force`], ); return; } const serviceLoaded = startResult.state.loadState.status === "loaded"; await emitStarted({ loaded: serviceLoaded }); } catch (err) { fail(`${params.serviceNoun} start failed: ${String(err)}`, params.renderStartHints()); } } export async function runServiceStop(params: { serviceNoun: string; service: GatewayService; opts?: DaemonLifecycleOptions; onNotLoaded?: (ctx: ServiceRecoveryContext) => Promise | null>; stopWhenNotLoaded?: boolean; }) { const json = Boolean(params.opts?.json); const { stdout, emit, fail } = createDaemonActionContext({ action: "stop", json }); const gatewayStopAudit = createServiceLifecycleMutationAudit({ serviceNoun: params.serviceNoun, action: "stop", }); const loaded = await resolveServiceLoadedOrFail({ serviceNoun: params.serviceNoun, service: params.service, fail, }); if (loaded === null) { return; } { const preflight = await getServiceActionPreflightFailure("stop"); if (preflight) { fail(`${params.serviceNoun} stop blocked: ${preflight.message}`, preflight.hints); return; } } if (!loaded) { if (params.stopWhenNotLoaded) { try { await params.service.stop({ env: process.env, stdout, disable: params.opts?.disable, onMutation: gatewayStopAudit, }); } catch (err) { fail(`${params.serviceNoun} stop failed: ${String(err)}`); return; } emit({ ok: true, result: "stopped", service: buildDaemonServiceSnapshot(params.service, false), }); return; } try { const handled = await params.onNotLoaded?.({ json, stdout, fail }); if (handled) { emit({ ok: true, result: handled.result, message: handled.message, warnings: handled.warnings, service: buildDaemonServiceSnapshot(params.service, false), }); if (!json && handled.message) { defaultRuntime.log(handled.message); } return; } } catch (err) { fail(`${params.serviceNoun} stop failed: ${String(err)}`); return; } emit({ ok: true, result: "not-loaded", message: `${params.serviceNoun} service ${params.service.notLoadedText}.`, service: buildDaemonServiceSnapshot(params.service, loaded), }); if (!json) { defaultRuntime.log(`${params.serviceNoun} service ${params.service.notLoadedText}.`); } return; } try { await params.service.stop({ env: process.env, stdout, disable: params.opts?.disable, onMutation: gatewayStopAudit, }); } catch (err) { fail(`${params.serviceNoun} stop failed: ${String(err)}`); return; } const finalLoaded = await resolveServiceLoadedOrFail({ serviceNoun: params.serviceNoun, service: params.service, fail, inspectionFailureMessage: `${params.serviceNoun} stop verification failed because service status is unknown`, }); if (finalLoaded === null) { return; } emit({ ok: true, result: "stopped", service: buildDaemonServiceSnapshot(params.service, finalLoaded), }); } export async function runServiceRestart(params: { serviceNoun: string; service: GatewayService; renderStartHints: () => string[]; opts?: DaemonLifecycleOptions; checkTokenDrift?: boolean; expectedPort?: number; beforeServiceMutation?: () => void; repairLoadedService?: ( ctx: ServiceStartRepairContext, ) => Promise | null>; postRestartCheck?: (ctx: RestartPostCheckContext) => Promise; onNotLoaded?: (ctx: ServiceRecoveryContext) => Promise | null>; restartOwnedProcess?: ( ctx: ServiceRecoveryContext, ) => Promise | null>; }): Promise { const json = Boolean(params.opts?.json); const { stdout, warnings, emit, fail } = createDaemonActionContext({ action: "restart", json }); const warn = json ? (message: string) => warnings.push(message) : undefined; const restartIntent = params.opts?.restartIntent; const gatewayRestartAudit = createServiceLifecycleMutationAudit({ serviceNoun: params.serviceNoun, action: "restart", }); let handledRecovery: ServiceRecoveryResult<"restarted"> | null = null; let handledRepair: ServiceRecoveryResult<"restarted"> | null = null; let recoveredLoadedState: boolean | null = null; let wroteRestartIntent = false; const prepareGatewayRestartIntent = async () => { if (params.serviceNoun !== "Gateway" || wroteRestartIntent) { return; } const runtime = await params.service.readRuntime(process.env).catch(() => null); assertGatewayServiceUpdateCurrent(); wroteRestartIntent = writeGatewayRestartIntentSync({ targetPid: runtime?.pid, reason: "gateway.restart", ...(restartIntent ? { intent: restartIntent } : {}), }); }; const clearPreparedRestartIntent = () => { if (wroteRestartIntent) { assertGatewayServiceUpdateCurrent(); clearGatewayRestartIntentSync(); wroteRestartIntent = false; } }; const emitScheduledRestart = ( restartStatus: ReturnType, serviceLoaded: boolean, ) => { return emitDaemonScheduledRestart({ json, emit, result: restartStatus.daemonActionResult, message: restartStatus.message, service: params.service, loaded: serviceLoaded, warnings, }); }; const loaded = await resolveServiceLoadedOrFail({ serviceNoun: params.serviceNoun, service: params.service, fail, acceptInstalledDefinition: true, }); if (loaded === null) { return false; } // Pre-flight config validation: check before any restart action (including // onNotLoaded which may send SIGUSR1 to an unmanaged process). (#35862) { const preflight = await getServiceActionPreflightFailure("restart"); if (preflight) { fail( preflight.hints ? `${params.serviceNoun} restart blocked: ${preflight.message}` : `${params.serviceNoun} aborted: config is invalid.\n${preflight.message}\n${formatInvalidConfigRecoveryHint()}`, preflight.hints, ); return false; } } if (params.restartOwnedProcess) { try { handledRecovery = await params.restartOwnedProcess({ json, stdout, warn, fail }); } catch (err) { fail(`${params.serviceNoun} restart failed: ${String(err)}`); return false; } } // Loaded services cross the native mutation boundary here. Not-loaded recovery // may still target a separately verified unmanaged listener. if (loaded && !handledRecovery) { params.beforeServiceMutation?.(); } if (!loaded && !handledRecovery) { try { handledRecovery = (await params.onNotLoaded?.({ json, stdout, warn, fail })) ?? null; } catch (err) { fail(`${params.serviceNoun} restart failed: ${String(err)}`); return false; } if (!handledRecovery) { await failServiceNotLoaded({ serviceNoun: params.serviceNoun, service: params.service, renderStartHints: params.renderStartHints, fail, }); return false; } if (handledRecovery.warnings?.length) { warnings.push(...handledRecovery.warnings); } recoveredLoadedState = handledRecovery.loaded ?? null; } if (loaded && !handledRecovery && params.repairLoadedService) { try { const { state, issues } = await inspectGatewayServiceStartRepair( params.service, { env: process.env }, params.expectedPort, ); if (issues.length > 0) { await prepareGatewayRestartIntent(); handledRepair = await params.repairLoadedService({ json, stdout, warn, fail, state, issues, }); if (!handledRepair) { clearPreparedRestartIntent(); fail( `${params.serviceNoun} service needs repair before restart: ${issues .map((issue) => issue.message) .join("; ")}`, [formatCliCommand("openclaw gateway install --force")], ); return false; } appendServiceLifecycleRepairAudit({ serviceNoun: params.serviceNoun, action: "restart", pid: state.runtime?.pid, }); if (handledRepair.warnings?.length) { warnings.push(...handledRepair.warnings); } } } catch (err) { clearPreparedRestartIntent(); const hints = params.renderStartHints(); fail(`${params.serviceNoun} repair failed: ${String(err)}`, hints); return false; } } if (loaded && !handledRecovery && params.checkTokenDrift) { // Check for token drift before restart (service token vs config token) try { const command = await params.service.readCommand(process.env); const serviceToken = command?.environment?.OPENCLAW_GATEWAY_TOKEN; const cfg = await readBestEffortConfig(); const driftEnv = { ...process.env, ...command?.environment, }; const configToken = await resolveGatewayTokenForDriftCheck({ cfg, env: driftEnv }); const driftIssue = checkTokenDrift({ serviceToken, configToken }); if (driftIssue) { const recovery = resolveDaemonInstallBlockMessage("gateway") ?? `Run \`${formatCliCommand("openclaw gateway install --force")}\` to refresh the service token source.`; const warning = `${driftIssue.message} ${recovery}`; warnings.push(warning); if (!json) { defaultRuntime.log(`\n⚠️ ${warning}\n`); } } } catch (err) { if (isGatewaySecretRefUnavailableError(err, "gateway.auth.token")) { const warning = "Unable to verify gateway token drift: gateway.auth.token SecretRef is configured but unavailable in this command path."; warnings.push(warning); if (!json) { defaultRuntime.log(`\n⚠️ ${warning}\n`); } } } } let postCheckFailed = false; try { let restartResult: GatewayServiceRestartResult | undefined; if (loaded && !handledRepair && !handledRecovery) { await prepareGatewayRestartIntent(); try { restartResult = await params.service.restart({ preserveDefinition: params.opts?.preserveDefinition, env: process.env, stdout, warn, onMutation: gatewayRestartAudit, }); } catch (err) { clearPreparedRestartIntent(); throw err; } } let restartStatus = describeGatewayServiceRestart( params.serviceNoun, restartResult ?? { outcome: "completed" }, ); if (restartStatus.scheduled) { return emitScheduledRestart(restartStatus, loaded || recoveredLoadedState === true); } if (params.postRestartCheck) { const postRestartResult = await params.postRestartCheck({ json, stdout, warnings, warn, // Definition repair alone does not record native activation. activationAccepted: restartResult?.outcome === "completed" || Boolean(handledRecovery), fail: (message, hints, result) => { postCheckFailed = true; fail(message, hints, result); }, }); if (postRestartResult) { restartStatus = describeGatewayServiceRestart(params.serviceNoun, postRestartResult); if (restartStatus.scheduled) { return emitScheduledRestart(restartStatus, loaded || recoveredLoadedState === true); } } } emit({ ok: true, result: "restarted", message: handledRecovery?.message ?? handledRepair?.message, service: buildDaemonServiceSnapshot(params.service, loaded || recoveredLoadedState === true), warnings: warnings.length ? warnings : undefined, }); const actionMessage = handledRecovery?.message ?? handledRepair?.message; if (!json && actionMessage) { defaultRuntime.log(actionMessage); } return true; } catch (err) { // A non-exiting runtime unwinds after emission; never replace that result. if (postCheckFailed) { throw err; } const hints = params.renderStartHints(); fail(`${params.serviceNoun} restart failed: ${String(err)}`, hints); return false; } }