scripts/openhands/p1-evaluate.ts

/**
 * P1 Go/Kill gate from openhands-p1-summary.json (§2.2 / §2.3).
 *
 * Env:
 *   P1_SUMMARY_JSON — default artifacts/openhands-p1-summary.json
 *   P1_MIN_AVG_SAVINGS — default 0.15 (15%)
 *   P1_MAX_RESOLVE_DROP_PP — default 3 (percentage points)
 *   P1_MIN_TWO_PATTERN_RATE — default 0.40
 */
import fs from "fs";
import path from "path";
import type { P1AggregateSummary } from "../../src/benchmark/openhands-aggregate.ts";

const summaryFile = path.resolve(process.env.P1_SUMMARY_JSON ?? "artifacts/openhands-p1-summary.json");
const minSavings = Number(process.env.P1_MIN_AVG_SAVINGS ?? "0.15");
const maxResolveDropPp = Number(process.env.P1_MAX_RESOLVE_DROP_PP ?? "3");
const minTwoPatternRate = Number(process.env.P1_MIN_TWO_PATTERN_RATE ?? "0.40");

if (!fs.existsSync(summaryFile)) {
  console.error(`P1 summary not found: ${summaryFile}`);
  process.exit(1);
}

const summary = JSON.parse(fs.readFileSync(summaryFile, "utf-8")) as P1AggregateSummary;

const savingsOk = summary.avg_savings_pct_c1 >= minSavings;
const resolveDrop = summary.resolve_drop_pp ?? 0;
const resolveOk = resolveDrop < maxResolveDropPp;
const patternOk = summary.two_plus_pattern_rate_c1 >= minTwoPatternRate;
const passed = savingsOk && resolveOk && patternOk;

const result = {
  gate: {
    minAvgSavings: minSavings,
    maxResolveDropPp,
    minTwoPatternRate,
    actualAvgSavings: summary.avg_savings_pct_c1,
    actualResolveDropPp: summary.resolve_drop_pp,
    actualTwoPatternRate: summary.two_plus_pattern_rate_c1,
    resolveRateByArm: summary.resolve_rate_by_arm,
    passed,
    checks: {
      savings: savingsOk,
      resolve_drop: resolveOk,
      two_plus_patterns: patternOk,
    },
    recommendation: passed ? "GO — proceed to C3 planning" : "HOLD — review arms or subset before L-tier spend",
  },
};

console.log(JSON.stringify(result, null, 2));
if (!passed) process.exit(1);