scripts/bench-cli-startup.ts

// Bench Cli Startup script supports OpenClaw repository automation.
import { execFileSync, spawn } from "node:child_process";
import {
  mkdirSync,
  mkdtempSync,
  readFileSync,
  readdirSync,
  realpathSync,
  rmSync,
  writeFileSync,
} from "node:fs";
import os from "node:os";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { expectDefined } from "../packages/normalization-core/src/expect.js";
import {
  assertCompatibleCliStartupExecutionModes,
  assertCompatibleCliStartupMemoryMetrics,
  CLI_RUNTIME_MEMORY_METRIC,
  type CliStartupExecutionMode,
  cliStartupMemoryMetric,
} from "./lib/cli-startup-memory-contract.mts";
import {
  inspectManagedProcessGroup,
  terminateManagedChild,
  waitForManagedProcessGroupExit,
} from "./lib/managed-child-process.mts";

type CommandCase = {
  id: string;
  name: string;
  args: string[];
  presets: readonly string[];
  stateScope?: "case" | "sample";
  expectedExitCodes?: readonly number[];
  expectedNonzeroOutputIncludes?: readonly string[];
  firstOutputBudgetMs?: number;
  exitBudgetMs?: number;
};

type Sample = {
  ms: number;
  firstOutputMs: number | null;
  maxRssMb: number | null;
  memory?: SampleMemory;
  exitCode: number | null;
  signal: string | null;
  startedAt?: string;
  endedAt?: string;
  timedOut?: boolean;
  stdoutTail?: string;
  stderrTail?: string;
};

type RssObservation = {
  pid: number;
  parentPid: number;
  matchesArguments: boolean;
  matchesInvocation: boolean;
  maxRssBytes: number | null;
};

type SampleMemory = {
  runtimePid: number | null;
  processes: Array<{
    pid: number;
    parentPid: number;
    role: "runtime" | "launcher" | "auxiliary" | "unresolved";
    metricKind: "process-high-water-rss";
    maxRssBytes: number | null;
  }>;
  error?: string;
};

type CaseRuns = {
  warmupSamples: Sample[];
  samples: Sample[];
};

type SummaryStats = {
  avg: number;
  p50: number;
  p95: number;
  min: number;
  max: number;
};

type CaseSummary = {
  sampleCount: number;
  durationMs: SummaryStats;
  firstOutputMs: SummaryStats | null;
  maxRssMb: SummaryStats | null;
  exitSummary: string;
};

type SuiteResult = {
  entry: string;
  executionMode?: CliStartupExecutionMode;
  memoryMetric?: string;
  cases: Array<{
    id: string;
    name: string;
    args: string[];
    expectedExitCodes?: number[];
    expectedNonzeroOutputIncludes?: string[];
    contract: {
      firstOutputBudgetMs: number | null;
      exitBudgetMs: number | null;
    } | null;
    warmupSamples?: Sample[];
    samples: Sample[];
    summary: CaseSummary;
  }>;
};

type BenchmarkReport = {
  primary: SuiteResult;
  secondary?: SuiteResult | null;
};

type CaseDelta = {
  id: string;
  name: string;
  durationAvgDeltaMs: number;
  durationAvgDeltaPct: number;
  maxRssAvgDeltaMb: number | null;
  maxRssAvgDeltaPct: number | null;
};

type BenchmarkComparison = {
  baseline: string;
  candidate: string;
  deltas: CaseDelta[];
};

type BenchmarkComparisonResult = {
  baseline: SuiteResult;
  candidate: SuiteResult;
  comparison: BenchmarkComparison;
};

type CliOptions = {
  cases: CommandCase[];
  compareBaseline?: string;
  compareCandidate?: string;
  entryPrimary: string;
  entrySecondary?: string;
  runs: number;
  warmup: number;
  timeoutMs: number;
  runtimeRss: boolean;
  json: boolean;
  output?: string;
  cpuProfDir?: string;
  heapProfDir?: string;
};

const DEFAULT_RUNS = 5;
const DEFAULT_WARMUP = 1;
const DEFAULT_TIMEOUT_MS = 30_000;
const DEFAULT_TIMEOUT_KILL_GRACE_MS = 1_000;
const TIMEOUT_KILL_GRACE_MS = resolveTimeoutKillGraceMs(process.env);
const DEFAULT_ENTRY = "openclaw.mjs";
const MAX_RSS_MARKER = "__OPENCLAW_MAX_RSS_KB__=";

type SampleTransport = {
  prefix: string[];
  binary: string;
  env: Record<string, string>;
  cwd: string;
};

function sampleTransport(): SampleTransport | undefined {
  const raw = process.env.OPENCLAW_BENCH_TRANSPORT_JSON;
  if (raw === undefined) {
    return undefined;
  }
  const value: unknown = JSON.parse(raw);
  if (!value || typeof value !== "object" || Array.isArray(value)) {
    throw new Error("Invalid benchmark transport");
  }
  const prefix = "prefix" in value ? value.prefix : undefined;
  const binary = "binary" in value ? value.binary : undefined;
  const env = "env" in value ? value.env : undefined;
  const cwd = "cwd" in value ? value.cwd : undefined;
  if (
    !Array.isArray(prefix) ||
    prefix.length === 0 ||
    !prefix.every(
      (part: unknown) => typeof part === "string" && part.length > 0 && !part.includes("\0"),
    ) ||
    typeof prefix[0] !== "string" ||
    !path.isAbsolute(prefix[0]) ||
    typeof binary !== "string" ||
    !path.isAbsolute(binary) ||
    binary.includes("\0") ||
    !env ||
    typeof env !== "object" ||
    Array.isArray(env)
  ) {
    throw new Error("Invalid benchmark transport");
  }
  const fixedEnv: Record<string, string> = {};
  for (const [key, entry] of Object.entries(env)) {
    if (!/^[A-Z_][A-Z0-9_]*$/u.test(key) || typeof entry !== "string" || entry.includes("\0")) {
      throw new Error("Invalid benchmark transport environment");
    }
    fixedEnv[key] = entry;
  }
  if (!fixedEnv.HOME || !path.isAbsolute(fixedEnv.HOME) || !fixedEnv.PATH) {
    throw new Error("Benchmark transport requires fixed HOME and PATH");
  }
  const directory = cwd === undefined ? fixedEnv.HOME : cwd;
  if (typeof directory !== "string" || !path.isAbsolute(directory) || directory.includes("\0")) {
    throw new Error("Invalid benchmark transport cwd");
  }
  return { prefix, binary, env: fixedEnv, cwd: directory };
}

function transportedCommand(
  transport: SampleTransport,
  args: string[],
  env: Record<string, string> = {},
  timeoutMs?: number,
): { command: string; args: string[] } {
  return {
    command: expectDefined(transport.prefix[0], "benchmark transport command"),
    args: [
      ...transport.prefix.slice(1),
      "/usr/bin/env",
      "-C",
      transport.cwd,
      "-i",
      ...Object.entries({ ...transport.env, ...env }).map(([key, value]) => `${key}=${value}`),
      ...(timeoutMs === undefined
        ? []
        : ["/usr/bin/timeout", "--signal=TERM", "--kill-after=1s", `${timeoutMs / 1000}s`]),
      transport.binary,
      ...args,
    ],
  };
}

function sampleFilesystem(
  transport: SampleTransport,
  operation: "create" | "prepare" | "remove",
  root?: string,
  config?: Record<string, unknown> | null,
  hook?: string,
): string {
  // Only the SUT opens these paths. Its replies are never read as paths on the runner.
  const launch = transportedCommand(transport, [
    "--input-type=module",
    "-e",
    `import fs from "node:fs"; import os from "node:os"; import path from "node:path";
const {operation,root,config,hook} = JSON.parse(fs.readFileSync(0,"utf8"));
if (operation === "create") process.stdout.write(fs.mkdtempSync(path.join(os.tmpdir(),"openclaw-cli-bench-home-")));
else if (operation === "prepare") {
  fs.mkdirSync(path.join(root,".openclaw"),{recursive:true});
  if (config) fs.writeFileSync(path.join(root,".openclaw/openclaw.json"),JSON.stringify(config)+"\\n");
  fs.writeFileSync(path.join(root,"measure-rss.mjs"),hook);
} else if (operation === "remove") fs.rmSync(root,{recursive:true,force:true});
else throw new Error("Invalid sample filesystem operation");`,
  ]);
  return execFileSync(launch.command, launch.args, {
    input: JSON.stringify({ operation, root, config, hook }),
    encoding: "utf8",
    timeout: 10_000,
    maxBuffer: 64 * 1024,
    env: { PATH: process.env.PATH },
  });
}

function createSampleRoot(transport?: SampleTransport): string {
  if (!transport) {
    return mkdtempSync(path.join(os.tmpdir(), "openclaw-cli-bench-home-"));
  }
  const root = sampleFilesystem(transport, "create");
  if (!path.isAbsolute(root) || /[\0\r\n]/u.test(root)) {
    throw new Error("Invalid SUT sample directory");
  }
  return root;
}

function removeSampleRoot(root: string, transport?: SampleTransport): void {
  if (transport) {
    sampleFilesystem(transport, "remove", root);
  } else {
    rmSync(root, { recursive: true, force: true });
  }
}

function resolveTimeoutKillGraceMs(env: NodeJS.ProcessEnv): number {
  const raw = env.VITEST ? env.OPENCLAW_TEST_CLI_STARTUP_TIMEOUT_KILL_GRACE_MS : undefined;
  if (!raw || !/^\d+$/u.test(raw)) {
    return DEFAULT_TIMEOUT_KILL_GRACE_MS;
  }
  const parsed = Number(raw);
  return Number.isSafeInteger(parsed) ? parsed : DEFAULT_TIMEOUT_KILL_GRACE_MS;
}
const VALUE_FLAGS = new Set([
  "--case",
  "--compare-baseline",
  "--compare-candidate",
  "--cpu-prof-dir",
  "--entry",
  "--entry-primary",
  "--entry-secondary",
  "--heap-prof-dir",
  "--output",
  "--preset",
  "--runs",
  "--timeout-ms",
  "--warmup",
]);
const BOOLEAN_FLAGS = new Set(["--help", "--json", "--runtime-rss"]);

const COMMAND_CASES: readonly CommandCase[] = [
  {
    id: "version",
    name: "--version",
    args: ["--version"],
    presets: ["startup", "response"],
    firstOutputBudgetMs: 1_000,
    exitBudgetMs: 2_000,
  },
  {
    id: "help",
    name: "--help",
    args: ["--help"],
    presets: ["startup", "response"],
    firstOutputBudgetMs: 1_000,
    exitBudgetMs: 2_000,
  },
  {
    id: "onboardHelp",
    name: "onboard --help",
    args: ["onboard", "--help"],
    presets: ["response"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "setupHelp",
    name: "setup --help",
    args: ["setup", "--help"],
    presets: ["response"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "configureHelp",
    name: "configure --help",
    args: ["configure", "--help"],
    presets: ["response"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "channelsAddHelp",
    name: "channels add --help",
    args: ["channels", "add", "--help"],
    presets: ["response"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "channelsParent",
    name: "channels",
    args: ["channels"],
    presets: ["response"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "doctorHelp",
    name: "doctor --help",
    args: ["doctor", "--help"],
    presets: ["response"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "modelsHelp",
    name: "models --help",
    args: ["models", "--help"],
    presets: ["response"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "pluginsHelp",
    name: "plugins --help",
    args: ["plugins", "--help"],
    presets: ["response"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "pluginsParent",
    name: "plugins",
    args: ["plugins"],
    presets: ["response"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "pluginsListJson",
    name: "plugins list --json",
    args: ["plugins", "list", "--json"],
    presets: ["response", "real"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "gatewayHelp",
    name: "gateway --help",
    args: ["gateway", "--help"],
    presets: ["response"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "agentsHelp",
    name: "agents --help",
    args: ["agents", "--help"],
    presets: ["response"],
    firstOutputBudgetMs: 3_500,
    exitBudgetMs: 8_000,
  },
  {
    id: "sessionsHelp",
    name: "sessions --help",
    args: ["sessions", "--help"],
    presets: ["response"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "tasksHelp",
    name: "tasks --help",
    args: ["tasks", "--help"],
    presets: ["response"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "messageHelp",
    name: "message --help",
    args: ["message", "--help"],
    presets: ["response"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "pairingHelp",
    name: "pairing --help",
    args: ["pairing", "--help"],
    presets: ["response"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "authHelp",
    name: "auth --help",
    args: ["auth", "--help"],
    presets: ["response"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "configHelp",
    name: "config --help",
    args: ["config", "--help"],
    presets: ["response"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "secretsHelp",
    name: "secrets --help",
    args: ["secrets", "--help"],
    presets: ["response"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "skillsHelp",
    name: "skills --help",
    args: ["skills", "--help"],
    presets: ["response"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "nodesHelp",
    name: "nodes --help",
    args: ["nodes", "--help"],
    presets: ["response"],
    firstOutputBudgetMs: 3_500,
    exitBudgetMs: 8_000,
  },
  {
    id: "directoryHelp",
    name: "directory --help",
    args: ["directory", "--help"],
    presets: ["response"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "sandboxHelp",
    name: "sandbox --help",
    args: ["sandbox", "--help"],
    presets: ["response"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "devicesParent",
    name: "devices",
    args: ["devices"],
    presets: ["response"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "mcpParent",
    name: "mcp",
    args: ["mcp"],
    presets: ["response"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "browserHelp",
    name: "browser --help",
    args: ["browser", "--help"],
    presets: ["response"],
    firstOutputBudgetMs: 1_500,
    exitBudgetMs: 3_000,
  },
  {
    id: "webhooksHelp",
    name: "webhooks --help",
    args: ["webhooks", "--help"],
    presets: ["response"],
    firstOutputBudgetMs: 2_500,
    exitBudgetMs: 6_000,
  },
  {
    id: "health",
    name: "health",
    args: ["health"],
    presets: ["startup", "real"],
    expectedExitCodes: [0, 1],
    expectedNonzeroOutputIncludes: ["Gateway target:"],
  },
  {
    id: "healthJson",
    name: "health --json",
    args: ["health", "--json"],
    presets: ["startup"],
    expectedExitCodes: [0, 1],
    expectedNonzeroOutputIncludes: ['"ok"', '"gateway_transport_error"'],
  },
  {
    id: "statusJson",
    name: "status --json",
    args: ["status", "--json"],
    presets: ["startup", "real"],
  },
  { id: "status", name: "status", args: ["status"], presets: ["startup", "real"] },
  { id: "sessions", name: "sessions", args: ["sessions"], presets: ["real"] },
  {
    id: "sessionsJson",
    name: "sessions --json",
    args: ["sessions", "--json"],
    presets: ["real"],
  },
  {
    id: "tasksJson",
    name: "tasks --json",
    args: ["tasks", "--json"],
    presets: ["real"],
  },
  {
    id: "tasksListJson",
    name: "tasks list --json",
    args: ["tasks", "list", "--json"],
    presets: ["real"],
  },
  {
    id: "tasksAuditJson",
    name: "tasks audit --json",
    args: ["tasks", "audit", "--json"],
    presets: ["real"],
  },
  {
    id: "agentsListJson",
    name: "agents list --json",
    args: ["agents", "list", "--json"],
    presets: ["real"],
  },
  {
    id: "gatewayStatus",
    name: "gateway status",
    args: ["gateway", "status"],
    presets: ["real"],
  },
  {
    id: "gatewayStatusJson",
    name: "gateway status --json",
    args: ["gateway", "status", "--json"],
    presets: ["real"],
  },
  {
    id: "gatewayHealthJson",
    name: "gateway health --json",
    args: ["gateway", "health", "--json"],
    presets: ["real"],
    expectedExitCodes: [0, 1],
    expectedNonzeroOutputIncludes: ['"ok"', '"gateway_transport_error"'],
  },
  {
    id: "gatewayHealthJsonWarmState",
    name: "gateway health --json (warm state)",
    args: ["gateway", "health", "--json"],
    presets: [],
    stateScope: "case",
  },
  {
    id: "gatewayHealthJsonFreshState",
    name: "gateway health --json (fresh state)",
    args: ["gateway", "health", "--json"],
    presets: [],
    stateScope: "sample",
  },
  {
    id: "configGetGatewayPort",
    name: "config get gateway.port",
    args: ["config", "get", "gateway.port"],
    presets: ["real"],
    expectedExitCodes: [0, 1],
    expectedNonzeroOutputIncludes: ["Config path not found: gateway.port"],
  },
] as const;

function parseFlagValue(flag: string): string | undefined {
  const idx = process.argv.indexOf(flag);
  if (idx === -1) {
    return undefined;
  }
  const value = process.argv[idx + 1];
  if (!value || value.startsWith("-")) {
    throw new Error(`${flag} requires a value`);
  }
  return value;
}

function hasFlag(flag: string): boolean {
  return process.argv.includes(flag);
}

function parseRepeatableFlag(flag: string): string[] {
  const values: string[] = [];
  for (let i = 0; i < process.argv.length; i += 1) {
    const value = process.argv[i + 1];
    if (process.argv[i] === flag && value && !value.startsWith("-")) {
      values.push(value);
    }
  }
  return values;
}

function validateCliArgs(argv: readonly string[] = process.argv.slice(2)): void {
  const seenSingleValueFlags = new Set<string>();
  for (let index = 0; index < argv.length; index += 1) {
    const arg = expectDefined(argv[index], `CLI benchmark argument at index ${index}`);
    if (VALUE_FLAGS.has(arg)) {
      if (arg !== "--case") {
        if (seenSingleValueFlags.has(arg)) {
          throw new Error(`${arg} was provided more than once`);
        }
        seenSingleValueFlags.add(arg);
      }
      const value = argv[index + 1];
      if (!value || value.startsWith("-")) {
        throw new Error(`${arg} requires a value`);
      }
      index += 1;
      continue;
    }
    if (BOOLEAN_FLAGS.has(arg)) {
      continue;
    }
    throw new Error(`Unknown argument: ${arg}`);
  }
}

function parsePositiveInt(raw: string | undefined, fallback: number, label = "value"): number {
  return parseIntegerOption(raw, fallback, label, 1);
}

function parseNonNegativeInt(raw: string | undefined, fallback: number, label = "value"): number {
  return parseIntegerOption(raw, fallback, label, 0);
}

// This runner is checked out from trusted main beside frozen candidates, whose
// root dependencies need not include current workspace packages.
function parseIntegerOption(
  raw: string | undefined,
  fallback: number,
  label: string,
  min: number,
): number {
  const value = raw?.trim();
  if (!value) {
    return fallback;
  }
  if (!/^\d+$/u.test(value)) {
    throw new Error(`${label} must be an integer >= ${min}; got ${JSON.stringify(raw)}`);
  }
  const parsed = Number(value);
  if (!Number.isSafeInteger(parsed) || parsed < min) {
    throw new Error(`${label} must be an integer >= ${min}; got ${JSON.stringify(raw)}`);
  }
  return parsed;
}

function parseGatewayPortEnv(raw: string | undefined): number {
  const value = raw?.trim();
  if (!value) {
    return 32123;
  }
  const bracketHostMatch = /^\[[^\]]+\]:(\d+)$/u.exec(value);
  if (bracketHostMatch) {
    return parsePositiveInt(bracketHostMatch[1], 32123, "OPENCLAW_GATEWAY_PORT");
  }
  if (value.startsWith("[") && value.endsWith("]")) {
    return 32123;
  }
  const colonCount = value.split(":").length - 1;
  if (colonCount > 1) {
    return 32123;
  }
  const portRaw = colonCount === 1 ? value.split(":")[1] : value;
  return parsePositiveInt(portRaw, 32123, "OPENCLAW_GATEWAY_PORT");
}

function parsePresets(raw: string | undefined): string[] {
  if (!raw) {
    return ["startup"];
  }
  const values = raw
    .split(",")
    .map((value) => value.trim())
    .filter(Boolean);
  if (values.includes("all")) {
    return ["startup", "real", "response"];
  }
  return values.length > 0 ? values : ["startup"];
}

function resolveCases(options: { presets: string[]; caseIds: string[] }): CommandCase[] {
  const byId = new Map(COMMAND_CASES.map((commandCase) => [commandCase.id, commandCase]));
  if (options.caseIds.length > 0) {
    const seenIds = new Set<string>();
    return options.caseIds.map((id) => {
      if (seenIds.has(id)) {
        throw new Error(`Duplicate --case "${id}"`);
      }
      seenIds.add(id);
      const commandCase = byId.get(id);
      if (!commandCase) {
        throw new Error(`Unknown --case "${id}"`);
      }
      return commandCase;
    });
  }
  return COMMAND_CASES.filter((commandCase) =>
    commandCase.presets.some((preset) => options.presets.includes(preset)),
  );
}

function median(values: number[]): number {
  if (values.length === 0) {
    return 0;
  }
  const sorted = [...values].toSorted((a, b) => a - b);
  const mid = Math.floor(sorted.length / 2);
  if (sorted.length % 2 === 0) {
    return (
      (expectDefined(sorted[mid - 1], "lower middle CLI benchmark sample") +
        expectDefined(sorted[mid], "upper middle CLI benchmark sample")) /
      2
    );
  }
  return expectDefined(sorted[mid], "middle CLI benchmark sample");
}

function percentile(values: number[], p: number): number {
  if (values.length === 0) {
    return 0;
  }
  const sorted = [...values].toSorted((a, b) => a - b);
  const index = Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length));
  return sorted[index] ?? 0;
}

function summarizeNumbers(values: number[]): SummaryStats {
  const total = values.reduce((sum, value) => sum + value, 0);
  const avg = values.length > 0 ? total / values.length : 0;
  const min = values.length > 0 ? Math.min(...values) : 0;
  const max = values.length > 0 ? Math.max(...values) : 0;
  return {
    avg,
    p50: median(values),
    p95: percentile(values, 95),
    min,
    max,
  };
}

function summarizeSamples(samples: Sample[]): CaseSummary {
  const durations = summarizeNumbers(samples.map((sample) => sample.ms));
  const firstOutputValues = samples
    .map((sample) => sample.firstOutputMs)
    .filter((value): value is number => typeof value === "number" && Number.isFinite(value));
  const rssValues = samples
    .map((sample) => sample.maxRssMb)
    .filter((value): value is number => typeof value === "number" && Number.isFinite(value));
  return {
    sampleCount: samples.length,
    durationMs: durations,
    firstOutputMs: firstOutputValues.length > 0 ? summarizeNumbers(firstOutputValues) : null,
    maxRssMb: rssValues.length > 0 ? summarizeNumbers(rssValues) : null,
    exitSummary: collectExitSummary(samples),
  };
}

function formatMs(value: number): string {
  return `${value.toFixed(1)}ms`;
}

function formatMb(value: number): string {
  return `${value.toFixed(1)}MiB`;
}

function collectExitSummary(samples: Sample[]): string {
  const buckets = new Map<string, number>();
  for (const sample of samples) {
    const key =
      sample.signal != null
        ? `signal:${sample.signal}`
        : `code:${sample.exitCode == null ? "null" : String(sample.exitCode)}`;
    buckets.set(key, (buckets.get(key) ?? 0) + 1);
  }
  return [...buckets.entries()].map(([key, count]) => `${key}x${count}`).join(", ");
}

function buildConfigFixture(commandCase: CommandCase): Record<string, unknown> | null {
  const usesSharedToken =
    commandCase.id === "gatewayHealthJsonWarmState" ||
    commandCase.id === "gatewayHealthJsonFreshState";
  if (
    commandCase.id !== "configGetGatewayPort" &&
    commandCase.id !== "gatewayHealthJson" &&
    !usesSharedToken &&
    commandCase.id !== "health" &&
    commandCase.id !== "healthJson"
  ) {
    return null;
  }
  const port = parseGatewayPortEnv(process.env.OPENCLAW_GATEWAY_PORT);
  return {
    gateway: {
      auth: { mode: usesSharedToken ? "token" : "none" },
      bind: "loopback",
      mode: "local",
      port,
    },
  };
}

function buildRssHook(tmpDir: string): string {
  const rssHookPath = path.join(tmpDir, "measure-rss.mjs");
  writeFileSync(
    rssHookPath,
    [
      "process.on('exit', () => {",
      "  const usage = typeof process.resourceUsage === 'function' ? process.resourceUsage() : null;",
      `  if (usage && typeof usage.maxRSS === 'number') console.error('${MAX_RSS_MARKER}' + String(usage.maxRSS));`,
      "});",
      "",
    ].join("\n"),
    "utf8",
  );
  return rssHookPath;
}

function parseMaxRssMb(stderr: string): number | null {
  const matches = [...stderr.matchAll(new RegExp(`^${MAX_RSS_MARKER}(\\d+)\\s*$`, "gm"))];
  const lastMatch = matches.at(-1);
  if (!lastMatch?.[1]) {
    return null;
  }
  return Number(lastMatch[1]) / 1024;
}

function buildRuntimeRssHook(tmpDir: string): string {
  const rssHookPath = path.join(tmpDir, "measure-rss.mjs");
  writeFileSync(
    rssHookPath,
    `import { writeFileSync, realpathSync } from "node:fs";
import { join } from "node:path";
import { isMainThread } from "node:worker_threads";
if (isMainThread) {
  const { directory, entries, args } = JSON.parse(process.env.OPENCLAW_BENCH_MEMORY);
  let entry;
  try {
    if (process.argv[1]) entry = realpathSync(process.argv[1]);
  } catch (error) {
    if (error.code !== "ENOENT") throw error;
  }
  const matchesArguments = JSON.stringify(process.argv.slice(2)) === JSON.stringify(args);
  const record = {
    pid: process.pid,
    parentPid: process.ppid,
    matchesArguments,
    matchesInvocation: entries.includes(entry) && matchesArguments,
    maxRssBytes: null,
  };
  const file = join(directory, String(process.pid) + ".json");
  writeFileSync(file, JSON.stringify(record));
  process.on("exit", () => {
    record.maxRssBytes = process.resourceUsage().maxRSS * 1024;
    writeFileSync(file, JSON.stringify(record));
  });
}
`,
    "utf8",
  );
  return rssHookPath;
}

function readSampleMemory(directory: string, entryPid: number | undefined): SampleMemory {
  const memory: SampleMemory = { runtimePid: null, processes: [] };
  try {
    const observations: RssObservation[] = readdirSync(directory).map((file) => {
      const value: unknown = JSON.parse(readFileSync(path.join(directory, file), "utf8"));
      if (
        typeof value !== "object" ||
        value === null ||
        !("pid" in value) ||
        typeof value.pid !== "number" ||
        !Number.isSafeInteger(value.pid) ||
        value.pid <= 0 ||
        file !== `${value.pid}.json` ||
        !("parentPid" in value) ||
        typeof value.parentPid !== "number" ||
        !Number.isSafeInteger(value.parentPid) ||
        value.parentPid < 0 ||
        !("matchesArguments" in value) ||
        typeof value.matchesArguments !== "boolean" ||
        !("matchesInvocation" in value) ||
        typeof value.matchesInvocation !== "boolean" ||
        !("maxRssBytes" in value) ||
        !(
          value.maxRssBytes === null ||
          (typeof value.maxRssBytes === "number" &&
            Number.isSafeInteger(value.maxRssBytes) &&
            value.maxRssBytes > 0)
        )
      ) {
        throw new Error("invalid process RSS observation");
      }
      return {
        pid: value.pid,
        parentPid: value.parentPid,
        matchesArguments: value.matchesArguments,
        matchesInvocation: value.matchesInvocation,
        maxRssBytes: value.maxRssBytes,
      };
    });
    memory.processes = observations.map((record) => ({
      pid: record.pid,
      parentPid: record.parentPid,
      role: record.matchesInvocation ? "unresolved" : "auxiliary",
      metricKind: "process-high-water-rss",
      maxRssBytes: record.maxRssBytes,
    }));
    if (observations.some((record) => record.matchesArguments && !record.matchesInvocation)) {
      throw new Error("unrecognized CLI entry with matching command arguments");
    }
    const invocation = observations.filter((record) => record.matchesInvocation);
    const entryObservation = invocation.find((record) => record.pid === entryPid);
    if (!entryObservation) {
      throw new Error("missing CLI entry process identity");
    }
    let current: RssObservation = entryObservation;
    const lineage = new Set<number>();
    // Respawns preserve CLI argv and the preload. Follow the unique invocation
    // chain, not exit order, RSS magnitude, or platform-specific ready flags.
    while (true) {
      if (lineage.has(current.pid)) {
        throw new Error("cyclic CLI process identity");
      }
      lineage.add(current.pid);
      if (current.maxRssBytes === null) {
        throw new Error(`missing process high-water RSS for CLI PID ${current.pid}`);
      }
      const parentPid = current.pid;
      const children = invocation.filter((record) => record.parentPid === parentPid);
      if (children.length > 1) {
        throw new Error("ambiguous CLI runtime identity: multiple matching children");
      }
      const child = children[0];
      if (!child) {
        break;
      }
      current = child;
    }
    if (lineage.size !== invocation.length) {
      throw new Error("disconnected CLI runtime identity");
    }
    memory.runtimePid = current.pid;
    for (const record of memory.processes) {
      if (lineage.has(record.pid)) {
        record.role = record.pid === current.pid ? "runtime" : "launcher";
      }
    }
  } catch (error) {
    memory.error = error instanceof Error ? error.message : String(error);
  }
  return memory;
}

function memoryInvocationEntries(entry: string): string[] {
  const resolvedEntry = realpathSync(entry);
  const entries = [resolvedEntry];
  // The wrapper's compile-cache handoff can enter dist directly.
  if (path.basename(resolvedEntry) === "openclaw.mjs") {
    for (const name of ["entry.js", "entry.mjs"]) {
      try {
        entries.push(realpathSync(path.join(path.dirname(resolvedEntry), "dist", name)));
      } catch (error) {
        if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
          throw error;
        }
      }
    }
  }
  return entries;
}

function nodeImportSpecifierForPath(filePath: string): string {
  return pathToFileURL(filePath).href;
}

function buildCpuOrHeapFlags(options: { cpuProfDir?: string; heapProfDir?: string }): string[] {
  const flags: string[] = [];
  if (options.cpuProfDir) {
    flags.push("--cpu-prof", "--cpu-prof-dir", options.cpuProfDir);
  }
  if (options.heapProfDir) {
    flags.push("--heap-prof", "--heap-prof-dir", options.heapProfDir);
  }
  return flags;
}

function appendLimited(current: string, chunk: Buffer | string, maxLength: number): string {
  const next = current + String(chunk);
  return next.length > maxLength ? next.slice(next.length - maxLength) : next;
}

async function runSample(params: {
  entry: string;
  commandCase: CommandCase;
  timeoutMs: number;
  runtimeRss: boolean;
  cpuProfDir?: string;
  heapProfDir?: string;
  rssHookPath: string;
  runRoot?: string;
}): Promise<Sample> {
  const transport = sampleTransport();
  const runRoot = params.runRoot ?? createSampleRoot(transport);
  const ownsRunRoot = params.runRoot == null;
  const stateDir = path.join(runRoot, ".openclaw");
  const configPath = path.join(stateDir, "openclaw.json");
  const configFixture = buildConfigFixture(params.commandCase);
  let rssHookPath = params.rssHookPath;
  if (transport) {
    sampleFilesystem(
      transport,
      "prepare",
      runRoot,
      configFixture,
      readFileSync(rssHookPath, "utf8"),
    );
    rssHookPath = path.join(runRoot, "measure-rss.mjs");
  } else if (configFixture) {
    mkdirSync(stateDir, { recursive: true });
    writeFileSync(configPath, `${JSON.stringify(configFixture, null, 2)}\n`, "utf8");
  }
  const nodeArgs = [
    ...(transport
      ? [
          "--import",
          `data:text/javascript,${encodeURIComponent(`process.chdir(${JSON.stringify(path.dirname(params.entry))});`)}`,
        ]
      : []),
    "--import",
    nodeImportSpecifierForPath(rssHookPath),
    ...buildCpuOrHeapFlags({
      cpuProfDir: params.cpuProfDir,
      heapProfDir: params.heapProfDir,
    }),
    params.entry,
    ...params.commandCase.args,
  ];
  const startedAt = new Date();
  const started = process.hrtime.bigint();
  let firstOutputMs: number | null = null;
  let stdout = "";
  let stderr = "";
  let settled = false;
  let timedOut = false;
  let forceKillAt: number | null = null;
  let forceKillTimer: ReturnType<typeof setTimeout> | null = null;
  const maxOutputLength = 32 * 1024 * 1024;
  const memoryDirectory = params.runtimeRss
    ? mkdtempSync(path.join(path.dirname(params.rssHookPath), "sample-"))
    : undefined;

  try {
    return await new Promise<Sample>((resolve) => {
      const sampleEnv = {
        HOME: runRoot,
        USERPROFILE: runRoot,
        OPENCLAW_HOME: runRoot,
        OPENCLAW_STATE_DIR: stateDir,
        OPENCLAW_CONFIG_PATH: configPath,
        OPENCLAW_HIDE_BANNER: "1",
        NO_COLOR: "1",
        FORCE_COLOR: "0",
        ...(memoryDirectory
          ? {
              OPENCLAW_BENCH_MEMORY: JSON.stringify({
                directory: memoryDirectory,
                entries: memoryInvocationEntries(params.entry),
                args: params.commandCase.args,
              }),
            }
          : {}),
      };
      const launch = transport
        ? transportedCommand(
            transport,
            nodeArgs,
            {
              ...sampleEnv,
              ...(process.env.OPENCLAW_GATEWAY_TOKEN
                ? { OPENCLAW_GATEWAY_TOKEN: process.env.OPENCLAW_GATEWAY_TOKEN }
                : {}),
              ...(process.env.OPENCLAW_GATEWAY_PORT
                ? { OPENCLAW_GATEWAY_PORT: process.env.OPENCLAW_GATEWAY_PORT }
                : {}),
            },
            params.timeoutMs,
          )
        : { command: process.execPath, args: nodeArgs };
      const proc = spawn(launch.command, launch.args, {
        cwd: process.cwd(),
        detached: process.platform !== "win32",
        env: transport ? { PATH: process.env.PATH } : { ...process.env, ...sampleEnv },
        stdio: ["ignore", "pipe", "pipe"],
      });

      const finish = (sample: Omit<Sample, "ms" | "firstOutputMs" | "maxRssMb">) => {
        if (settled) {
          return;
        }
        settled = true;
        if (forceKillTimer) {
          clearTimeout(forceKillTimer);
          forceKillTimer = null;
        }
        const ms = Number(process.hrtime.bigint() - started) / 1e6;
        const memory = memoryDirectory ? readSampleMemory(memoryDirectory, proc.pid) : undefined;
        const runtimeRss = memory?.processes.find(
          (record) => record.role === "runtime",
        )?.maxRssBytes;
        resolve({
          ms,
          firstOutputMs,
          maxRssMb: memory
            ? runtimeRss == null
              ? null
              : runtimeRss / 1024 / 1024
            : parseMaxRssMb(stderr),
          ...(memory ? { memory } : {}),
          startedAt: startedAt.toISOString(),
          endedAt: new Date().toISOString(),
          ...(timedOut ? { timedOut } : {}),
          ...sample,
        });
      };

      const markFirstOutput = () => {
        if (firstOutputMs == null) {
          firstOutputMs = Number(process.hrtime.bigint() - started) / 1e6;
        }
      };

      const timeout = setTimeout(() => {
        timedOut = true;
        signalSampleProcess(proc, "SIGTERM");
        forceKillAt = Date.now() + TIMEOUT_KILL_GRACE_MS;
        forceKillTimer = setTimeout(() => {
          signalSampleProcess(proc, "SIGKILL");
        }, TIMEOUT_KILL_GRACE_MS).unref?.();
      }, params.timeoutMs);
      timeout.unref?.();

      proc.stdout?.on("data", (chunk) => {
        markFirstOutput();
        stdout = appendLimited(stdout, chunk, maxOutputLength);
      });
      proc.stderr?.on("data", (chunk) => {
        markFirstOutput();
        stderr = appendLimited(stderr, chunk, maxOutputLength);
      });
      proc.once("error", (error) => {
        clearTimeout(timeout);
        stderr = appendLimited(
          stderr,
          error instanceof Error ? error.message : String(error),
          maxOutputLength,
        );
        finish({
          exitCode: null,
          signal: null,
          stdoutTail: tailLines(stdout, 20),
          stderrTail: tailLines(stderr, 20),
        });
      });
      proc.once("close", (code, signal) => {
        clearTimeout(timeout);
        const complete = () =>
          finish({
            exitCode: code,
            signal,
            ...(code === 0 && signal == null
              ? {}
              : {
                  stdoutTail: tailLines(stdout, 20),
                  stderrTail: tailLines(stderr, 20),
                }),
          });
        if (timedOut && isSampleProcessGroupAlive(proc)) {
          void finishAfterTimeoutCleanup({
            complete,
            forceKillAt,
            proc,
          });
          return;
        }
        complete();
      });
    });
  } finally {
    if (memoryDirectory) {
      rmSync(memoryDirectory, { recursive: true, force: true });
    }
    if (ownsRunRoot) {
      removeSampleRoot(runRoot, transport);
    }
  }
}

async function finishAfterTimeoutCleanup(params: {
  complete: () => void;
  forceKillAt: number | null;
  proc: ReturnType<typeof spawn>;
}): Promise<void> {
  const graceRemainingMs =
    params.forceKillAt === null
      ? TIMEOUT_KILL_GRACE_MS
      : Math.max(0, params.forceKillAt - Date.now());
  if (graceRemainingMs > 0) {
    await waitForSampleProcessGroupExit(params.proc, graceRemainingMs);
  }
  if (isSampleProcessGroupAlive(params.proc)) {
    signalSampleProcess(params.proc, "SIGKILL");
  }
  await waitForSampleProcessGroupExit(params.proc, TIMEOUT_KILL_GRACE_MS);
  params.complete();
}

function signalSampleProcess(proc: ReturnType<typeof spawn>, signal: NodeJS.Signals): void {
  if (!proc.pid) {
    return;
  }
  const handleSignalError = (error: unknown) => {
    const code = (error as NodeJS.ErrnoException | undefined)?.code;
    if (code !== "ESRCH" && code !== "EPERM") {
      throw error;
    }
  };
  terminateManagedChild(proc, signal, {
    onChildSignalError: handleSignalError,
    onProcessGroupSignalError: handleSignalError,
    processGroupFallback: "never",
    useWindowsTaskkill: false,
  });
}

function isSampleProcessGroupAlive(proc: ReturnType<typeof spawn>): boolean {
  return inspectManagedProcessGroup(proc, { errorPolicy: "alive-on-eperm" }) === "live";
}

function waitForSampleProcessGroupExit(
  proc: ReturnType<typeof spawn>,
  timeoutMs: number,
): Promise<boolean> {
  return waitForManagedProcessGroupExit(proc, timeoutMs, { errorPolicy: "alive-on-eperm" });
}

async function runCase(params: {
  entry: string;
  commandCase: CommandCase;
  runs: number;
  warmup: number;
  timeoutMs: number;
  runtimeRss: boolean;
  cpuProfDir?: string;
  heapProfDir?: string;
  rssHookPath: string;
}): Promise<CaseRuns> {
  const warmupSamples: Sample[] = [];
  const samples: Sample[] = [];
  const totalRuns = params.warmup + params.runs;
  const transport = sampleTransport();
  const caseRunRoot =
    params.commandCase.stateScope === "case" ? createSampleRoot(transport) : undefined;
  try {
    for (let i = 0; i < totalRuns; i += 1) {
      const sample = await runSample({ ...params, runRoot: caseRunRoot });
      if (i < params.warmup) {
        warmupSamples.push(sample);
        continue;
      }
      samples.push(sample);
    }
    return { warmupSamples, samples };
  } finally {
    if (caseRunRoot) {
      removeSampleRoot(caseRunRoot, transport);
    }
  }
}

function tailLines(value: string, maxLines: number): string {
  return value.split(/\r?\n/).filter(Boolean).slice(-maxLines).join("\n");
}

function printSuite(result: SuiteResult): void {
  console.log(`Entry: ${result.entry}`);
  console.log(`RSS metric: ${cliStartupMemoryMetric(result)}`);
  for (const commandCase of result.cases) {
    const { durationMs, firstOutputMs, maxRssMb, exitSummary } = commandCase.summary;
    const rssSummary =
      maxRssMb == null
        ? "rss=n/a"
        : `rss(avg=${formatMb(maxRssMb.avg)} p50=${formatMb(maxRssMb.p50)} p95=${formatMb(maxRssMb.p95)})`;
    const firstOutputSummary =
      firstOutputMs == null
        ? "first-output=n/a"
        : `first-output(avg=${formatMs(firstOutputMs.avg)} p50=${formatMs(
            firstOutputMs.p50,
          )} p95=${formatMs(firstOutputMs.p95)})`;
    console.log(
      `${commandCase.name.padEnd(24)} avg=${formatMs(durationMs.avg)} p50=${formatMs(
        durationMs.p50,
      )} p95=${formatMs(durationMs.p95)} min=${formatMs(durationMs.min)} max=${formatMs(
        durationMs.max,
      )} ${firstOutputSummary} ${rssSummary} exits=[${exitSummary}]`,
    );
  }
  console.log("");
}

function printDelta(primary: SuiteResult, secondary: SuiteResult): void {
  const deltas = buildCaseDeltas(primary, secondary);
  console.log("Delta (secondary - primary, avg)");
  for (const delta of deltas) {
    const durationDelta = delta.durationAvgDeltaMs;
    const durationPct = delta.durationAvgDeltaPct;
    const durationSign = durationDelta > 0 ? "+" : "";
    let line = `${delta.name.padEnd(24)} ${durationSign}${formatMs(durationDelta)} (${durationSign}${durationPct.toFixed(1)}%)`;
    if (delta.maxRssAvgDeltaMb != null && delta.maxRssAvgDeltaPct != null) {
      const rssDelta = delta.maxRssAvgDeltaMb;
      const rssPct = delta.maxRssAvgDeltaPct;
      const rssSign = rssDelta > 0 ? "+" : "";
      line += ` rss ${rssSign}${formatMb(rssDelta)} (${rssSign}${rssPct.toFixed(1)}%)`;
    }
    console.log(line);
  }
}

function buildCaseDeltas(primary: SuiteResult, secondary: SuiteResult): CaseDelta[] {
  assertCompatibleCliStartupExecutionModes(primary, secondary);
  assertCompatibleCliStartupMemoryMetrics(primary, secondary);
  const primaryById = new Map(primary.cases.map((commandCase) => [commandCase.id, commandCase]));
  const deltas: CaseDelta[] = [];
  for (const commandCase of secondary.cases) {
    const baseline = primaryById.get(commandCase.id);
    if (!baseline) {
      continue;
    }
    const durationDelta = commandCase.summary.durationMs.avg - baseline.summary.durationMs.avg;
    const durationPct =
      baseline.summary.durationMs.avg > 0
        ? (durationDelta / baseline.summary.durationMs.avg) * 100
        : 0;
    const rssDelta =
      baseline.summary.maxRssMb && commandCase.summary.maxRssMb
        ? commandCase.summary.maxRssMb.avg - baseline.summary.maxRssMb.avg
        : null;
    const rssPct =
      rssDelta != null && baseline.summary.maxRssMb && baseline.summary.maxRssMb.avg > 0
        ? (rssDelta / baseline.summary.maxRssMb.avg) * 100
        : null;
    deltas.push({
      id: commandCase.id,
      name: commandCase.name,
      durationAvgDeltaMs: durationDelta,
      durationAvgDeltaPct: durationPct,
      maxRssAvgDeltaMb: rssDelta,
      maxRssAvgDeltaPct: rssPct,
    });
  }
  return deltas;
}

export function collectFailedSamples(result: SuiteResult): string[] {
  const failures: string[] = [];
  for (const commandCase of result.cases) {
    if (commandCase.samples.length === 0) {
      failures.push(`${result.entry} ${commandCase.id}: no measured samples`);
    }
    for (const [sampleKind, samples] of [
      ["warmup", commandCase.warmupSamples ?? []],
      ["sample", commandCase.samples],
    ] as const) {
      for (const [sampleIndex, sample] of samples.entries()) {
        const label = `${result.entry} ${commandCase.id} ${sampleKind} ${sampleIndex + 1}`;
        const expectedExitCodes = new Set(commandCase.expectedExitCodes ?? [0]);
        if (sample.timedOut === true) {
          failures.push(`${label}: timed out`);
        } else if (sample.signal !== null) {
          failures.push(`${label}: exited via signal ${sample.signal}`);
        } else if (!expectedExitCodes.has(sample.exitCode ?? -1)) {
          failures.push(`${label}: exited with code ${String(sample.exitCode)}`);
        } else if (sample.maxRssMb === null) {
          failures.push(
            `${label}: did not report max RSS${sample.memory?.error ? ` (${sample.memory.error})` : ""}`,
          );
        } else if (sample.exitCode !== 0) {
          const output = `${sample.stdoutTail ?? ""}\n${sample.stderrTail ?? ""}`;
          const missing = (commandCase.expectedNonzeroOutputIncludes ?? []).filter(
            (snippet) => !output.includes(snippet),
          );
          if (missing.length > 0) {
            failures.push(
              `${label}: exited with expected code ${String(
                sample.exitCode,
              )} but output did not match expected clean-state markers (${missing.join(", ")})`,
            );
          }
        }
      }
    }
  }
  return failures;
}

async function buildSuiteResult(params: {
  entry: string;
  executionMode: CliStartupExecutionMode;
  options: CliOptions;
  rssHookPath: string;
}): Promise<SuiteResult> {
  const cases = [];
  for (const commandCase of params.options.cases) {
    const { warmupSamples, samples } = await runCase({
      entry: params.entry,
      commandCase,
      runs: params.options.runs,
      warmup: params.options.warmup,
      timeoutMs: params.options.timeoutMs,
      runtimeRss: params.options.runtimeRss,
      cpuProfDir: params.options.cpuProfDir,
      heapProfDir: params.options.heapProfDir,
      rssHookPath: params.rssHookPath,
    });
    cases.push({
      id: commandCase.id,
      name: commandCase.name,
      args: commandCase.args,
      ...(commandCase.expectedExitCodes && commandCase.expectedExitCodes.some((code) => code !== 0)
        ? { expectedExitCodes: [...commandCase.expectedExitCodes] }
        : {}),
      ...(commandCase.expectedNonzeroOutputIncludes
        ? { expectedNonzeroOutputIncludes: [...commandCase.expectedNonzeroOutputIncludes] }
        : {}),
      contract:
        commandCase.firstOutputBudgetMs != null || commandCase.exitBudgetMs != null
          ? {
              firstOutputBudgetMs: commandCase.firstOutputBudgetMs ?? null,
              exitBudgetMs: commandCase.exitBudgetMs ?? null,
            }
          : null,
      warmupSamples,
      samples,
      summary: summarizeSamples(samples),
    });
  }
  return {
    entry: params.entry,
    executionMode: params.executionMode,
    ...(params.options.runtimeRss ? { memoryMetric: CLI_RUNTIME_MEMORY_METRIC } : {}),
    cases,
  };
}

function parseOptions(): CliOptions {
  const presets = parsePresets(parseFlagValue("--preset"));
  const cases = resolveCases({
    presets,
    caseIds: parseRepeatableFlag("--case"),
  });
  return {
    cases,
    compareBaseline: parseFlagValue("--compare-baseline"),
    compareCandidate: parseFlagValue("--compare-candidate"),
    entryPrimary: parseFlagValue("--entry-primary") ?? parseFlagValue("--entry") ?? DEFAULT_ENTRY,
    entrySecondary: parseFlagValue("--entry-secondary"),
    runs: parsePositiveInt(parseFlagValue("--runs"), DEFAULT_RUNS, "--runs"),
    warmup: parseNonNegativeInt(parseFlagValue("--warmup"), DEFAULT_WARMUP, "--warmup"),
    timeoutMs: parsePositiveInt(parseFlagValue("--timeout-ms"), DEFAULT_TIMEOUT_MS, "--timeout-ms"),
    runtimeRss: hasFlag("--runtime-rss"),
    json: hasFlag("--json"),
    output: parseFlagValue("--output"),
    cpuProfDir: parseFlagValue("--cpu-prof-dir"),
    heapProfDir: parseFlagValue("--heap-prof-dir"),
  };
}

function printUsage(): void {
  console.log(`OpenClaw CLI benchmark

Usage:
  pnpm tsx scripts/bench-cli-startup.ts [options]

Options:
  --preset <startup|real|response|all>
                               Command preset to run (default: startup)
  --case <id>                  Specific case id to run; repeatable
  --entry <path>               Primary entry file (default: openclaw.mjs)
  --entry-secondary <path>     Secondary entry file for avg delta comparison
  --runs <n>                   Measured runs per case (default: ${DEFAULT_RUNS})
  --warmup <n>                 Warmup runs per case (default: ${DEFAULT_WARMUP})
  --timeout-ms <ms>            Per-run timeout (default: ${DEFAULT_TIMEOUT_MS})
  --output <path>              Write machine-readable JSON to a file
  --compare-baseline <path>    Read a saved JSON report as the baseline
  --compare-candidate <path>   Read a saved JSON report as the candidate and print deltas
  --cpu-prof-dir <dir>         Write V8 CPU profiles for each run
  --heap-prof-dir <dir>        Write V8 heap profiles for each run
  --runtime-rss                Attribute RSS to the CLI runtime (default: legacy last marker)
  --json                       Emit machine-readable JSON
  --help                       Show this text

Case ids:
  ${COMMAND_CASES.map((commandCase) => `${commandCase.id} (${commandCase.name})`).join("\n  ")}
`);
}

function readBenchmarkReport(filePath: string): BenchmarkReport {
  return JSON.parse(readFileSync(filePath, "utf8")) as BenchmarkReport;
}

function writeJsonOutput(filePath: string, value: unknown): void {
  mkdirSync(path.dirname(filePath), { recursive: true });
  writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
}

function readBenchmarkComparison(
  baselinePath: string,
  candidatePath: string,
): BenchmarkComparisonResult {
  const baseline = readBenchmarkReport(baselinePath);
  const candidate = readBenchmarkReport(candidatePath);
  return {
    baseline: baseline.primary,
    candidate: candidate.primary,
    comparison: {
      baseline: baselinePath,
      candidate: candidatePath,
      deltas: buildCaseDeltas(baseline.primary, candidate.primary),
    },
  };
}

async function main(): Promise<void> {
  validateCliArgs();
  if (hasFlag("--help")) {
    printUsage();
    return;
  }

  const options = parseOptions();
  const transport = sampleTransport();
  if (transport && options.runtimeRss) {
    throw new Error("Cross-user runtime RSS sampling is not supported");
  }
  if (transport && (options.cpuProfDir || options.heapProfDir)) {
    throw new Error(
      "Cross-user CLI profiles must be collected through the SUT diagnostic exporter",
    );
  }
  if (options.compareBaseline || options.compareCandidate) {
    if (!options.compareBaseline || !options.compareCandidate) {
      throw new Error("--compare-baseline and --compare-candidate must be provided together");
    }
    const { baseline, candidate, comparison } = readBenchmarkComparison(
      options.compareBaseline,
      options.compareCandidate,
    );
    if (options.output) {
      writeJsonOutput(options.output, comparison);
    }
    if (options.json) {
      console.log(JSON.stringify(comparison, null, 2));
      return;
    }
    printDelta(baseline, candidate);
    return;
  }
  const tmpDir = mkdtempSync(path.join(os.tmpdir(), "openclaw-cli-bench-"));
  const rssHookPath = options.runtimeRss ? buildRuntimeRssHook(tmpDir) : buildRssHook(tmpDir);
  try {
    const primary = await buildSuiteResult({
      entry: options.entryPrimary,
      executionMode: transport ? "transport" : "native",
      options,
      rssHookPath,
    });
    const secondary = options.entrySecondary
      ? await buildSuiteResult({
          entry: options.entrySecondary,
          executionMode: transport ? "transport" : "native",
          options,
          rssHookPath,
        })
      : undefined;

    const report = {
      node: process.version,
      runs: options.runs,
      warmup: options.warmup,
      timeoutMs: options.timeoutMs,
      cpuProfDir: options.cpuProfDir ?? null,
      heapProfDir: options.heapProfDir ?? null,
      primary,
      secondary: secondary ?? null,
    };
    const failures = [
      ...collectFailedSamples(primary),
      ...(secondary ? collectFailedSamples(secondary) : []),
    ];

    if (options.output) {
      writeJsonOutput(options.output, report);
    }

    if (options.json) {
      console.log(JSON.stringify(report, null, 2));
      if (failures.length > 0) {
        process.exitCode = 1;
        for (const failure of failures) {
          console.error(`[startup-bench] ${failure}`);
        }
      }
      return;
    }

    console.log(`Node: ${process.version}`);
    console.log(`Runs per case: ${options.runs}`);
    console.log(`Warmup runs per case: ${options.warmup}`);
    console.log(`Timeout: ${options.timeoutMs}ms`);
    if (options.cpuProfDir) {
      console.log(`CPU profiles: ${options.cpuProfDir}`);
    }
    if (options.heapProfDir) {
      console.log(`Heap profiles: ${options.heapProfDir}`);
    }
    console.log("");

    console.log("Primary entry");
    printSuite(primary);
    if (secondary) {
      console.log("Secondary entry");
      printSuite(secondary);
      printDelta(primary, secondary);
    }

    if (failures.length > 0) {
      process.exitCode = 1;
      console.error("\nFailed startup benchmark samples:");
      for (const failure of failures) {
        console.error(`- ${failure}`);
      }
    }
  } finally {
    rmSync(tmpDir, { recursive: true, force: true });
  }
}

export const testing = {
  buildConfigFixture,
  collectFailedSamples,
  nodeImportSpecifierForPath,
  parseGatewayPortEnv,
  parseNonNegativeInt,
  parsePositiveInt,
  validateCliArgs,
};

if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
  await main().catch((error: unknown) => {
    console.error(error instanceof Error ? error.message : String(error));
    process.exit(1);
  });
}