scripts/bench-openhands.ts

/**
 * OpenHands benchmark: traces → adapt → optimize (optional) → report.
 *
 * Env:
 *   BENCH_BASE_URL, BENCH_API_KEY (required unless OPENHANDS_ARM=C0)
 *   OPENHANDS_TRACE_DIR — recursive *.jsonl (supports C1/, C2/ subdirs)
 *   OPENHANDS_ARM — C0 | C1 | C2 | C3 (default C1). C0 = metrics only, no TCG call
 *   BENCH_MODE — fast | deep | auto (default fast; C2 often deep)
 *   SWE_RESULTS_FILE — JSON map instance_id → { C0|C1|C2: { resolved, usd, ... } }
 *   SUBMIT_FEEDBACK — true to POST /api/v1/feedback when trace_id + swe result exist
 *   BENCH_OUTPUT_FILE — default artifacts/openhands-reports/bench-<ARM>.json
 */
import fs from "fs";
import path from "path";
import { adaptOpenHandsTrace, parseOpenHandsJsonl } from "../src/benchmark/openhands-adapter.ts";
import {
  discoverTraceFiles,
  estimateUsdFromLogs,
  loadSweResults,
  runOptimizeOnLogs,
  submitFeedback,
  summarizeReport,
  type TraceBenchRow,
} from "../src/benchmark/openhands-run.ts";
import { ROI_PATTERN_TYPES } from "../src/benchmark/openhands-run.ts";

const baseUrl = process.env.BENCH_BASE_URL ?? "http://127.0.0.1:3000";
const apiKey = process.env.BENCH_API_KEY ?? "";
const arm = (process.env.OPENHANDS_ARM ?? "C1").toUpperCase();
const mode =
  process.env.BENCH_MODE ??
  (arm === "C2" ? "deep" : arm === "C3" ? "fast" : "fast");
const traceDir = path.resolve(process.env.OPENHANDS_TRACE_DIR ?? "scripts/fixtures");
const sweResults = loadSweResults(process.env.SWE_RESULTS_FILE);
const submitFeedbackEnabled = (process.env.SUBMIT_FEEDBACK ?? "false").toLowerCase() === "true";
const outputFile = path.resolve(
  process.env.BENCH_OUTPUT_FILE ?? `artifacts/openhands-reports/bench-${arm}.json`
);

const skipOptimize = arm === "C0";

async function main() {
  if (!skipOptimize && !apiKey) {
    console.error("BENCH_API_KEY is required unless OPENHANDS_ARM=C0");
    process.exit(1);
  }

  const armSubdir = path.join(traceDir, arm);
  const searchDir =
    fs.existsSync(armSubdir) && fs.statSync(armSubdir).isDirectory() ? armSubdir : traceDir;
  const files = discoverTraceFiles(searchDir, arm).filter((f) => f.arm === arm);
  if (files.length === 0) {
    console.error(`No .jsonl traces under ${traceDir}`);
    process.exit(1);
  }

  const traces: TraceBenchRow[] = [];
  const latencies: number[] = [];

  for (const { file, arm: fileArm, instance_id } of files) {
    const effectiveArm = fileArm || arm;
    const events = parseOpenHandsJsonl(fs.readFileSync(file, "utf-8"));
    const adapted = adaptOpenHandsTrace(events);
    const swe = sweResults[instance_id]?.[effectiveArm];

    if (adapted.logs.length === 0) {
      traces.push({
        arm: effectiveArm,
        instance_id,
        file: path.relative(process.cwd(), file),
        log_count: 0,
        metrics: adapted.metrics,
        optimize_status: 0,
        optimize_latency_ms: 0,
        pattern_types: [],
        pattern_hit_count: 0,
        savings_percentage: null,
        trace_id: null,
        usd_est_from_logs: 0,
        tokens_from_logs: 0,
        swe,
        error: "no_logs_after_adapt",
      });
      continue;
    }

    const tokensFromLogs = adapted.logs.reduce(
      (s, l) => s + l.prompt_tokens + l.completion_tokens,
      0
    );
    const usdEst = estimateUsdFromLogs(adapted.logs);

    const traceId = `openhands-${effectiveArm}-${instance_id}-${Date.now()}`;

    if (skipOptimize) {
      traces.push({
        arm: effectiveArm,
        instance_id,
        file: path.relative(process.cwd(), file),
        log_count: adapted.logs.length,
        metrics: adapted.metrics,
        optimize_status: 204,
        optimize_latency_ms: 0,
        pattern_types: [],
        pattern_hit_count: 0,
        savings_percentage: null,
        trace_id: traceId,
        usd_est_from_logs: Number(usdEst.toFixed(6)),
        tokens_from_logs: tokensFromLogs,
        swe,
      });
      continue;
    }

    const opt = await runOptimizeOnLogs(baseUrl, apiKey, adapted.logs, mode, traceId);
    if (opt.latency > 0) latencies.push(opt.latency);
    const roiPatterns = opt.pattern_types.filter((p) => ROI_PATTERN_TYPES.has(p));

    traces.push({
      arm: effectiveArm,
      instance_id,
      file: path.relative(process.cwd(), file),
      log_count: adapted.logs.length,
      metrics: adapted.metrics,
      optimize_status: opt.status,
      optimize_latency_ms: opt.latency,
      pattern_types: opt.pattern_types,
      pattern_hit_count: roiPatterns.length,
      savings_percentage: opt.savings_percentage,
      trace_id: opt.trace_id ?? traceId,
      usd_est_from_logs: Number(usdEst.toFixed(6)),
      tokens_from_logs: tokensFromLogs,
      swe,
      error: opt.error,
    });

    if (submitFeedbackEnabled && opt.trace_id && swe && apiKey) {
      await submitFeedback(baseUrl, apiKey, {
        trace_id: opt.trace_id,
        success: Boolean(swe.resolved),
        score: swe.resolved ? 1 : 0,
        metadata: { arm: effectiveArm, instance_id, ...swe },
      });
    }
  }

  const report = {
    generated_at: new Date().toISOString(),
    base_url: baseUrl,
    arm,
    mode,
    trace_dir: traceDir,
    summary: summarizeReport(traces, latencies),
    traces,
  };

  fs.mkdirSync(path.dirname(outputFile), { recursive: true });
  fs.writeFileSync(outputFile, `${JSON.stringify(report, null, 2)}\n`, "utf-8");
  console.log(JSON.stringify(report.summary, null, 2));
  console.error(`Wrote ${outputFile}`);
}

void main();