scripts/bench-openhands-gate.ts

import fs from "fs";
import path from "path";

type OpenHandsReport = {
  summary?: {
    p95_latency_ms?: number;
    optimize_success_rate?: number;
    pattern_hit_rate?: number;
    trace_files?: number;
  };
};

const reportFile = path.resolve(process.env.BENCH_REPORT_FILE ?? "artifacts/openhands-bench-report.json");
const maxP95 = Number(process.env.BENCH_GATE_MAX_P95_MS ?? "2000");
const minSuccessRate = Number(process.env.BENCH_GATE_MIN_SUCCESS_RATE ?? "0.99");
const minPatternHitRate = Number(process.env.BENCH_GATE_MIN_PATTERN_HIT_RATE ?? "0");

if (!fs.existsSync(reportFile)) {
  console.error(`OpenHands bench report not found: ${reportFile}`);
  process.exit(1);
}

const parsed = JSON.parse(fs.readFileSync(reportFile, "utf-8")) as OpenHandsReport;
const summary = parsed.summary;

if (!summary) {
  console.error("Invalid report: missing summary");
  process.exit(1);
}

const passed =
  (summary.p95_latency_ms ?? Infinity) <= maxP95 &&
  (summary.optimize_success_rate ?? 0) >= minSuccessRate &&
  (summary.pattern_hit_rate ?? 0) >= minPatternHitRate;

console.log(
  JSON.stringify(
    {
      gate: {
        maxP95,
        minSuccessRate,
        minPatternHitRate,
        actualP95: summary.p95_latency_ms,
        actualSuccessRate: summary.optimize_success_rate,
        actualPatternHitRate: summary.pattern_hit_rate,
        passed,
      },
    },
    null,
    2
  )
);

if (!passed) process.exit(1);