// Gateway service lifecycle runners, including unmanaged-process fallbacks and restart health checks. import { theme } from "../../../packages/terminal-core/src/theme.js"; import { resolveGatewayServiceProbeHosts } from "../../daemon/gateway-service-probe-hosts.js"; import { assertGatewayServiceUpdateCurrent, assertGatewayServiceFallbackAllowed, } from "../../daemon/service-update-authority.js"; import { resolveGatewayService } from "../../daemon/service.js"; import { findInstalledSystemdGatewayScope, restartSystemdService, stopSystemdService, } from "../../daemon/systemd.js"; import { type GatewayLockIdentity, readActiveGatewayLockIdentity, readActiveGatewayLockPort, } from "../../infra/gateway-lock.js"; import { readGatewayOwnerLease } from "../../infra/gateway-owner-lease.js"; import { formatGatewayPidList, signalVerifiedGatewayPidSync, } from "../../infra/gateway-processes.js"; import { assertGatewayServiceMutationAllowed, formatExternalSupervisorActionRequired, isGatewayExternallySupervised, resolveGatewayServiceMutationError, } from "../../infra/gateway-supervision.js"; import { probePortUsage } from "../../infra/ports-probe.js"; import type { GatewayRestartIntent } from "../../infra/restart-intent.js"; import { resolveGatewayRestartDeferralTimeoutMs } from "../../infra/restart.js"; import { defaultRuntime } from "../../runtime.js"; import { formatCliCommand } from "../command-format.js"; import { isTerminalInteractive, NON_INTERACTIVE_GATEWAY_STOP_MESSAGE, } from "../terminal-interactivity.js"; import { recoverInstalledLaunchAgent } from "./launchd-recovery.js"; import { appendGatewayLifecycleAudit, createGatewayLifecycleMutationAudit, } from "./lifecycle-audit.js"; import { resolveGatewayConfigPorts, resolveGatewayLifecycleContext } from "./lifecycle-context.js"; import { runServiceRestart, runServiceStart, runServiceStop, runServiceUninstall, } from "./lifecycle-core.js"; import { runSafeGatewayRestart, resolveGatewayRestartIntentOptions, } from "./lifecycle-safe-restart.js"; import { resolveVerifiedGatewayListenerPids, signalGatewayRestart } from "./lifecycle-unmanaged.js"; import { createDaemonActionContext, createNullWriter } from "./response.js"; import { DEFAULT_RESTART_HEALTH_ATTEMPTS, DEFAULT_RESTART_HEALTH_DELAY_MS, formatGatewayRestartFailure, renderGatewayPortHealthDiagnostics, renderRestartDiagnostics, terminateStaleGatewayPids, waitForGatewayHealthyListener, waitForGatewayHealthyRestart, } from "./restart-health.js"; import { renderGatewayServiceStartHints } from "./shared.js"; import { verifyGatewayStartReadiness } from "./start-health.js"; import { repairLoadedGatewayServiceForStart } from "./start-repair.js"; import type { DaemonLifecycleOptions } from "./types.js"; const POST_RESTART_HEALTH_ATTEMPTS = DEFAULT_RESTART_HEALTH_ATTEMPTS; const POST_RESTART_HEALTH_DELAY_MS = DEFAULT_RESTART_HEALTH_DELAY_MS; const WINDOWS_POST_RESTART_HEALTH_TIMEOUT_MS = 180_000; function postRestartHealthAttempts(): number { return process.platform === "win32" ? Math.ceil(WINDOWS_POST_RESTART_HEALTH_TIMEOUT_MS / POST_RESTART_HEALTH_DELAY_MS) : POST_RESTART_HEALTH_ATTEMPTS; } async function handleSystemScopeSystemdGateway( action: "stop", ): Promise<{ result: "stopped"; message: string } | null>; async function handleSystemScopeSystemdGateway( action: "restart", ): Promise<{ result: "restarted"; message: string } | null>; async function handleSystemScopeSystemdGateway( action: "stop" | "restart", ): Promise<{ result: "stopped" | "restarted"; message: string } | null> { if (process.platform !== "linux") { return null; } const installed = await findInstalledSystemdGatewayScope(process.env).catch(() => null); if (installed?.scope !== "system") { return null; } const stdout = createNullWriter(); if (action === "stop") { await stopSystemdService({ stdout, env: process.env, onMutation: createGatewayLifecycleMutationAudit({ action: "stop" }), }); return { result: "stopped", message: `Gateway stopped via system-scope systemd unit ${installed.unitName}.`, }; } await restartSystemdService({ stdout, env: process.env, onMutation: createGatewayLifecycleMutationAudit({ action: "restart" }), }); return { result: "restarted", message: `Gateway restarted via system-scope systemd unit ${installed.unitName}.`, }; } async function stopGatewayWithoutServiceManager( port: number, lockOwnerPid: number | undefined, serviceContext?: Parameters[0], ) { const managed = await handleSystemScopeSystemdGateway("stop"); if (managed) { return managed; } const listenerPids = resolveVerifiedGatewayListenerPids(port); // Listener discovery needs lsof, which minimal containers omit. The gateway // lock already names the verified owner of this port, so signal it instead of // reporting the gateway as not running while it keeps serving. const pids = listenerPids.length > 0 ? listenerPids : lockOwnerPid ? [lockOwnerPid] : []; if (pids.length === 0) { const probeHosts = await resolveGatewayServiceProbeHosts(serviceContext ?? {}); const portUsage = await probePortUsage(port, probeHosts); if (portUsage !== "free") { throw new Error( portUsage === "busy" ? `Port ${port} is in use but the owning process could not be identified. Run ${formatCliCommand("openclaw gateway status --deep")} to diagnose.` : `Could not determine whether port ${port} is still in use, so the gateway cannot be confirmed stopped. Run ${formatCliCommand("openclaw gateway status --deep")} to diagnose.`, ); } return null; } for (const pid of pids) { signalVerifiedGatewayPidSync(pid, "SIGTERM"); appendGatewayLifecycleAudit({ action: "stop", source: "cli", mode: "sigterm", pid, }); } return { result: "stopped" as const, message: `Gateway stop signal sent to unmanaged process${pids.length === 1 ? "" : "es"} on port ${port}: ${formatGatewayPidList(pids)}.`, }; } async function resolveRestartListenerHealthWait(restartIntent: GatewayRestartIntent | undefined) { let drainTimeoutMs: number | undefined; if (restartIntent?.force) { drainTimeoutMs = 0; } else if (typeof restartIntent?.waitMs === "number" && Number.isFinite(restartIntent.waitMs)) { drainTimeoutMs = restartIntent.waitMs > 0 ? Math.floor(restartIntent.waitMs) : undefined; } else { drainTimeoutMs = resolveGatewayRestartDeferralTimeoutMs(); } const replacementHealthAttempts = postRestartHealthAttempts(); if (drainTimeoutMs === undefined) { return { attempts: replacementHealthAttempts, waitIndefinitelyForPreviousOwner: true, timeoutSeconds: Math.round((replacementHealthAttempts * POST_RESTART_HEALTH_DELAY_MS) / 1000), }; } const attempts = replacementHealthAttempts + Math.ceil(drainTimeoutMs / POST_RESTART_HEALTH_DELAY_MS); return { attempts, waitIndefinitelyForPreviousOwner: false, timeoutSeconds: Math.round((attempts * POST_RESTART_HEALTH_DELAY_MS) / 1000), }; } async function restartUnmanaged(port: number, intent?: GatewayRestartIntent, allowSystem = true) { const managed = allowSystem ? await handleSystemScopeSystemdGateway("restart") : null; if (managed) { return managed; } return await signalGatewayRestart(port, { restartIntent: intent, enforceRestartConfig: true, processLabel: "unmanaged", auditSource: "cli", }); } function isGatewaySignalRestartResult( result: Awaited>, ): result is NonNullable>> { return result !== null && "pid" in result && typeof result.pid === "number"; } async function runExternalSupervisorRestart(opts: DaemonLifecycleOptions): Promise { const { emit, fail } = createDaemonActionContext({ action: "restart", json: Boolean(opts.json) }); const restartIntent = resolveGatewayRestartIntentOptions(opts); const lockIdentity = await readActiveGatewayLockIdentity().catch(() => undefined); if (!lockIdentity?.ownerId) { // Owner IDs and targeted restart semantics shipped together. Older Gateways // ignore target fields, so fail before a candidate can mutate their state. fail( "Gateway restart failed: the active Gateway lock predates targeted restart ownership; update the running Gateway before retrying", ); return false; } if (opts.safe) { return await runSafeGatewayRestart(opts, { ...lockIdentity, ownerId: lockIdentity.ownerId }); } let signaled: Awaited>; try { signaled = await signalGatewayRestart(lockIdentity.port, { restartIntent, enforceRestartConfig: false, processLabel: "externally supervised", requireLockIdentity: true, auditSource: "supervisor", }); } catch (err) { fail(`Gateway restart failed: ${String(err)}`); return false; } if (!signaled) { fail( `No verified gateway process is listening on port ${lockIdentity.port}. ${formatExternalSupervisorActionRequired("start the gateway")}`, ); return false; } const healthWait = await resolveRestartListenerHealthWait(restartIntent); const health = await waitForGatewayHealthyListener({ port: lockIdentity.port, attempts: healthWait.attempts, delayMs: POST_RESTART_HEALTH_DELAY_MS, previousLockIdentity: signaled.previousLockIdentity, waitIndefinitelyForPreviousOwner: healthWait.waitIndefinitelyForPreviousOwner, }); if (!health.healthy) { const message = `Gateway restart timed out after ${healthWait.timeoutSeconds}s waiting for health checks.`; fail(message, renderGatewayPortHealthDiagnostics(health)); return false; } emit({ ok: true, result: signaled.result, message: signaled.message, }); if (!opts.json) { defaultRuntime.log(signaled.message); } return true; } /** Uninstall the managed Gateway service after stopping it. */ export async function runDaemonUninstall(opts: DaemonLifecycleOptions = {}) { assertGatewayServiceMutationAllowed("uninstall the gateway service"); return await runServiceUninstall({ serviceNoun: "Gateway", service: resolveGatewayService(), opts, stopBeforeUninstall: true, assertNotLoadedAfterUninstall: true, }); } /** Start the managed Gateway service, repairing stale service definitions when possible. */ export async function runDaemonStart(opts: DaemonLifecycleOptions = {}) { assertGatewayServiceMutationAllowed("start the gateway"); const service = resolveGatewayService(); const expectedPort = (await resolveGatewayConfigPorts()).explicit; return await runServiceStart({ serviceNoun: "Gateway", service, renderStartHints: renderGatewayServiceStartHints, onNotLoaded: process.platform === "darwin" ? async () => { const recovered = await recoverInstalledLaunchAgent({ result: "started" }); if (recovered) { appendGatewayLifecycleAudit({ action: "start", source: "cli", mode: "launchd-bootstrap", }); } return recovered; } : undefined, repairLoadedService: async ({ json, stdout, warn, state, issues }) => await repairLoadedGatewayServiceForStart({ service, json, stdout, warn, state, issues, }), postStartCheck: ({ fail, warnings }) => verifyGatewayStartReadiness({ service, expectedPort, resolveContext: () => resolveGatewayLifecycleContext(service), fail, warnings, }), expectedPort, opts, }); } /** Stop the managed Gateway service or verified unmanaged listener fallback. */ export async function runDaemonStop(opts: DaemonLifecycleOptions = {}) { if (!isTerminalInteractive() && !opts.force) { const { fail } = createDaemonActionContext({ action: "stop", json: Boolean(opts.json) }); fail(NON_INTERACTIVE_GATEWAY_STOP_MESSAGE); return; } assertGatewayServiceUpdateCurrent(); assertGatewayServiceMutationAllowed("stop the gateway"); const service = resolveGatewayService(); return await runServiceStop({ serviceNoun: "Gateway", service, opts, stopWhenNotLoaded: process.platform === "darwin" && Boolean(opts.disable), onNotLoaded: async ({ stdout }) => { if (process.platform === "linux") { const runtime = await service.readRuntime(process.env).catch(() => null); if (runtime?.status === "running") { // systemd can run a disabled unit with Restart=always. Stop it through // systemctl so a process-level SIGTERM cannot trigger a respawn. await service.stop({ env: process.env, stdout, onMutation: createGatewayLifecycleMutationAudit({ action: "stop" }), }); return { result: "stopped" }; } } assertGatewayServiceFallbackAllowed("unmanaged stop"); // An unmanaged run loop keeps its lock port across config edits, so use it // for discovery the way restart already does; otherwise a valid port // override makes the running gateway look like it is already stopped. const lock = await readActiveGatewayLockIdentity().catch(() => undefined); const ctx = lock ? null : await resolveGatewayLifecycleContext(service).catch(() => null); const port = lock?.port ?? ctx?.port ?? (await resolveGatewayConfigPorts()).fallback; return await stopGatewayWithoutServiceManager(port, lock?.pid, ctx ?? undefined); }, }); } /** Restart the Gateway service or a verified unmanaged listener, then prove health. */ export async function runDaemonRestart(opts: DaemonLifecycleOptions = {}): Promise { const preserveDefinition = Boolean(opts.preserveDefinition); if (preserveDefinition) { assertGatewayServiceMutationAllowed("restart the gateway"); if (opts.safe) { throw new Error("--preserve-definition requires a native restart without --safe"); } } if (opts.skipDeferral && !opts.safe) { throw new Error("--skip-deferral requires --safe"); } if (isGatewayExternallySupervised()) { assertGatewayServiceFallbackAllowed("external-supervisor restart"); return await runExternalSupervisorRestart(opts); } if (opts.safe) { assertGatewayServiceFallbackAllowed("safe RPC restart"); return await runSafeGatewayRestart(opts); } const jsonOutput = Boolean(opts.json); const service = resolveGatewayService(); let restartedWithoutServiceManager = false; let unmanagedPreviousLockIdentity: GatewayLockIdentity | undefined; const restartIntent = resolveGatewayRestartIntentOptions(opts); const { explicit: configuredPort, fallback: fallbackPort } = await resolveGatewayConfigPorts(); let managedRestartContext = await resolveGatewayLifecycleContext( service, preserveDefinition, ).catch(async (error: unknown) => { if (preserveDefinition) { throw error; } return { port: fallbackPort, env: process.env }; }); let managedRestartPort = preserveDefinition ? managedRestartContext.port : (configuredPort ?? managedRestartContext.port); // An unmanaged run loop keeps its lock port across in-process restarts, even // when config changes underneath it. Use that port for both the signal and // health proof or a valid CLI/env override looks like a failed restart. let unmanagedPort = (await readActiveGatewayLockPort().catch(() => undefined)) ?? managedRestartPort; const restartHealthAttempts = postRestartHealthAttempts(); const restartWaitMs = restartHealthAttempts * POST_RESTART_HEALTH_DELAY_MS; const restartWaitSeconds = Math.round(restartWaitMs / 1000); let unmanagedRestartHealthAttempts = restartHealthAttempts; let unmanagedRestartWaitIndefinitely = false; let unmanagedRestartWaitSeconds = restartWaitSeconds; return await runServiceRestart({ serviceNoun: "Gateway", service, renderStartHints: renderGatewayServiceStartHints, opts: { ...opts, ...(restartIntent ? { restartIntent } : {}), }, checkTokenDrift: true, expectedPort: configuredPort, beforeServiceMutation: () => assertGatewayServiceMutationAllowed("restart the gateway"), restartOwnedProcess: preserveDefinition ? undefined : async () => { const owner = readGatewayOwnerLease(); if (!owner || owner.state === "dead" || owner.mode !== "foreground") { return null; } assertGatewayServiceFallbackAllowed("foreground owner restart"); if (owner.state !== "live") { throw new Error( `Cannot verify foreground Gateway owner pid ${owner.pid} on ${owner.host}`, ); } unmanagedPort = owner.port; const ready = await waitForGatewayHealthyListener({ port: owner.port, env: process.env, attempts: restartHealthAttempts, delayMs: POST_RESTART_HEALTH_DELAY_MS, }); if (!ready.healthy) { throw new Error( `Foreground Gateway owner pid ${owner.pid} is still starting or unhealthy on port ${owner.port}; it was left running`, ); } const handled = await signalGatewayRestart(owner.port, { restartIntent, enforceRestartConfig: true, processLabel: "foreground", requireLockIdentity: true, auditSource: "cli", ownerLease: owner, env: process.env, }); if (!handled) { throw new Error( `Foreground Gateway owner pid ${owner.pid} no longer listens on port ${owner.port}`, ); } restartedWithoutServiceManager = true; unmanagedPreviousLockIdentity = handled.previousLockIdentity; const healthWait = await resolveRestartListenerHealthWait(restartIntent); unmanagedRestartHealthAttempts = healthWait.attempts; unmanagedRestartWaitIndefinitely = healthWait.waitIndefinitelyForPreviousOwner; unmanagedRestartWaitSeconds = healthWait.timeoutSeconds; return handled; }, repairLoadedService: preserveDefinition ? undefined : async ({ json, stdout, warn, state, issues }) => { const result = await repairLoadedGatewayServiceForStart({ action: "restart", service, json, stdout, warn, state, issues, }); // Repair rewrites the service definition, so the old command environment // no longer identifies where the restarted gateway publishes readiness. managedRestartContext = await resolveGatewayLifecycleContext(service); managedRestartPort = configuredPort ?? managedRestartContext.port; return result; }, onNotLoaded: async () => { assertGatewayServiceFallbackAllowed("unmanaged restart"); if (preserveDefinition) { return null; } const mutationError = resolveGatewayServiceMutationError("restart the gateway"); if (process.platform === "darwin" && !mutationError) { const recovered = await recoverInstalledLaunchAgent({ result: "restarted" }); if (recovered) { appendGatewayLifecycleAudit({ action: "restart", source: "cli", mode: "launchd-bootstrap", }); return recovered; } } const handled = await restartUnmanaged(unmanagedPort, restartIntent, !mutationError); if (handled) { restartedWithoutServiceManager = true; if (isGatewaySignalRestartResult(handled) && handled.previousLockIdentity) { unmanagedPreviousLockIdentity = handled.previousLockIdentity; const healthWait = await resolveRestartListenerHealthWait(restartIntent); unmanagedRestartHealthAttempts = healthWait.attempts; unmanagedRestartWaitIndefinitely = healthWait.waitIndefinitelyForPreviousOwner; unmanagedRestartWaitSeconds = healthWait.timeoutSeconds; } return handled; } if (mutationError) { throw mutationError; } return null; }, postRestartCheck: async ({ warnings, fail, stdout, warn, activationAccepted: accepted }) => { let activationAccepted = accepted; if (restartedWithoutServiceManager) { // Unmanaged restarts have no service-manager state to watch; use listener health and, // when targeted delivery required it, prove the previous lock owner was replaced. const health = await waitForGatewayHealthyListener({ port: unmanagedPort, env: process.env, attempts: unmanagedRestartHealthAttempts, delayMs: POST_RESTART_HEALTH_DELAY_MS, ...(unmanagedPreviousLockIdentity ? { previousLockIdentity: unmanagedPreviousLockIdentity, waitIndefinitelyForPreviousOwner: unmanagedRestartWaitIndefinitely, } : {}), }); if (health.healthy) { return undefined; } const diagnostics = renderGatewayPortHealthDiagnostics(health); const timeoutLine = `Timed out after ${unmanagedRestartWaitSeconds}s waiting for gateway port ${unmanagedPort} to become healthy.`; if (!jsonOutput) { defaultRuntime.log(theme.warn(timeoutLine)); for (const line of diagnostics) { defaultRuntime.log(theme.muted(line)); } } else { warnings.push(timeoutLine); warnings.push(...diagnostics); } fail( `Gateway restart timed out after ${unmanagedRestartWaitSeconds}s waiting for health checks.`, [formatCliCommand("openclaw gateway status --deep"), formatCliCommand("openclaw doctor")], activationAccepted ? "restart-health-failed" : undefined, ); throw new Error("unreachable after gateway restart health failure"); } const waitForHealthy = async () => await waitForGatewayHealthyRestart({ service, port: managedRestartPort, attempts: restartHealthAttempts, delayMs: POST_RESTART_HEALTH_DELAY_MS, env: managedRestartContext.env, supervisorKeepsAlive: process.platform === "darwin", }); let health = await waitForHealthy(); if (!health.healthy && health.staleGatewayPids.length > 0) { // On Windows service restarts can leave stale listeners behind; kill verified stale // Gateway pids once, restart again, then re-run the same health proof. const staleMsg = `Found stale gateway process(es): ${health.staleGatewayPids.join(", ")}.`; warnings.push(staleMsg); if (!jsonOutput) { defaultRuntime.log(theme.warn(staleMsg)); defaultRuntime.log(theme.muted("Stopping stale process(es) and retrying restart...")); } const terminated = await terminateStaleGatewayPids(health.staleGatewayPids, { env: managedRestartContext.env, assertCurrent: assertGatewayServiceUpdateCurrent, }); const currentOwner = readGatewayOwnerLease({ env: managedRestartContext.env }); if (terminated.length > 0 && (!currentOwner || currentOwner.state === "dead")) { const retryRestart = await service.restart({ preserveDefinition, env: process.env, stdout, warn, onMutation: createGatewayLifecycleMutationAudit({ action: "restart" }), }); if (retryRestart.outcome === "scheduled") { return retryRestart; } activationAccepted = true; } health = await waitForHealthy(); } if (health.healthy) { return undefined; } const diagnostics = renderRestartDiagnostics(health); const failure = formatGatewayRestartFailure({ health, port: managedRestartPort, defaultTimeoutSeconds: restartWaitSeconds, }); const runningNoPortLine = health.runtime.status === "running" && health.portUsage.status === "free" ? `Gateway process is running but port ${managedRestartPort} is still free (startup hang/crash loop or very slow VM startup).` : null; if (!jsonOutput) { defaultRuntime.log(theme.warn(failure.statusLine)); if (runningNoPortLine) { defaultRuntime.log(theme.warn(runningNoPortLine)); } for (const line of diagnostics) { defaultRuntime.log(theme.muted(line)); } } else { warnings.push(failure.statusLine); if (runningNoPortLine) { warnings.push(runningNoPortLine); } warnings.push(...diagnostics); } fail( failure.failMessage, [formatCliCommand("openclaw gateway status --deep"), formatCliCommand("openclaw doctor")], activationAccepted ? "restart-health-failed" : undefined, ); throw new Error("unreachable after gateway restart failure"); }, }); }