apps/api/src/__tests__/judge-failure-export.test.ts

import { readFileSync } from "node:fs";
import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";

import JSZip from "jszip";
import { describe, expect, it } from "vitest";

import { QueueRunner } from "../queue/runner.js";
import { SqliteStore } from "../queue/store.js";

const repoRoot = fileURLToPath(new URL("../../../../", import.meta.url));

function readRepoFile(path: string): string {
  return readFileSync(resolve(repoRoot, path), "utf8");
}

describe("judge failure export behavior", () => {
  it("builds the task package zip before marking a judge-quality failure", () => {
    const runner = readRepoFile("apps/api/src/queue/runner.ts");

    const zipExportIndex = runner.indexOf('stage = "zip_export"');
    const judgeFailureIndex = runner.indexOf("if (!passed)");

    expect(zipExportIndex).toBeGreaterThan(-1);
    expect(judgeFailureIndex).toBeGreaterThan(-1);
    expect(zipExportIndex).toBeLessThan(judgeFailureIndex);
  });

  it("allows download-all to include failed items when an export zip exists", () => {
    const analysisRoute = readRepoFile("apps/api/src/routes/analysis.ts");

    expect(analysisRoute).not.toContain('it.status === "completed" &&');
    expect(analysisRoute).toContain("it.export_zip_path &&");
  });

  it("caps judge refinement at five rounds and exposes per-round scores", () => {
    const runner = readRepoFile("apps/api/src/queue/runner.ts");
    const analysisRoute = readRepoFile("apps/api/src/routes/analysis.ts");

    expect(runner).toContain("const MAX_JUDGE_ITERATIONS = 5");
    expect(runner).toContain("judge-result-iter-${iter}.json");
    expect(analysisRoute).toContain("readJudgeHistory");
    expect(analysisRoute).toContain("judgeHistory");
  });

  it("keeps a downloadable export when Judge throws before returning a verdict", async () => {
    const dir = mkdtempSync(resolve(tmpdir(), "rl-auto-judge-http-fail-"));
    try {
      const uploadPath = resolve(dir, "recording.zip");
      const original = new JSZip();
      original.file("network.json", JSON.stringify([{ method: "POST", url: "https://example.test/api" }]));
      writeFileSync(uploadPath, await original.generateAsync({ type: "nodebuffer" }));

      const store = new SqliteStore(resolve(dir, "tasks.db"));
      const job = store.createJob({ requestedMode: "api" });
      const created = store.createItem({
        jobId: job.id,
        ord: 1,
        filename: "recording.zip",
        uploadPath,
        detectedMode: "api",
      });
      const item = store.takeNextQueuedItem(job.id);
      expect(item?.id).toBe(created.id);

      const logger = {
        child: () => logger,
        info: () => undefined,
        warn: () => undefined,
        error: () => undefined,
      } as any;
      const runner = new QueueRunner({
        store,
        logger,
        config: {
          exportDir: resolve(dir, "exports"),
          itemMaxAttempts: 1,
          globalLlmConcurrency: 0,
          upstreamProxy: "",
          defaultApiKey: "unused",
          defaultBaseURL: "https://api.deepseek.com",
          defaultApiMode: "chat",
          defaultModel: "deepseek-v4-pro",
        } as any,
      });
      (runner as any).runAnalyzer = async () => ({
        mode: "api",
        parsed: {},
        evidence: { type: "http", requests: [] },
        flags: [],
        result: {
          recommended_instruction: "Send the API request.",
          recommended_groups: [
            {
              id: "group_1",
              title: "Send request",
              intent: "Send request",
              callIds: [],
              apiCalls: [],
            },
          ],
          recommended_rubrics: [
            {
              name: "request_sent",
              description: "The expected API request is sent.",
              category: "process",
              checker_key: "request_sent",
              scoring: { 0: "Request missing", 1: "Request present" },
            },
          ],
        },
      });
      (runner as any).withLlmSlot = async () => {
        throw Object.assign(
          new Error("HTTP 403 - API key is not active or has been deleted"),
          { status: 403 },
        );
      };

      await (runner as any).runItem(item, job.id);

      const failed = store.getItem(item!.id);
      expect(failed?.status).toBe("failed");
      expect(failed?.last_stage).toBe("iteration_1:judge");
      expect(failed?.export_zip_path).toBeTruthy();
      expect(existsSync(failed!.export_zip_path!)).toBe(true);
      expect(failed?.task_package_path).toBeTruthy();
      expect(existsSync(failed!.task_package_path!)).toBe(true);
      store.close();
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  it("feeds optimized iteration memory into later analyzer rounds and persists it", async () => {
    const dir = mkdtempSync(resolve(tmpdir(), "rl-auto-iteration-memory-"));
    try {
      const uploadPath = resolve(dir, "recording.zip");
      const original = new JSZip();
      original.file("network.json", JSON.stringify([{ method: "POST", url: "https://example.test/api" }]));
      writeFileSync(uploadPath, await original.generateAsync({ type: "nodebuffer" }));

      const store = new SqliteStore(resolve(dir, "tasks.db"));
      const job = store.createJob({ requestedMode: "api" });
      const created = store.createItem({
        jobId: job.id,
        ord: 1,
        filename: "recording.zip",
        uploadPath,
        detectedMode: "api",
      });
      const item = store.takeNextQueuedItem(job.id);
      expect(item?.id).toBe(created.id);

      const logger = {
        child: () => logger,
        info: () => undefined,
        warn: () => undefined,
        error: () => undefined,
      } as any;
      const runner = new QueueRunner({
        store,
        logger,
        config: {
          exportDir: resolve(dir, "exports"),
          itemMaxAttempts: 1,
          globalLlmConcurrency: 0,
          upstreamProxy: "",
          defaultApiKey: "unused",
          defaultBaseURL: "https://api.deepseek.com",
          defaultApiMode: "chat",
          defaultModel: "deepseek-v4-pro",
        } as any,
      });
      const analyzerFeedback: string[] = [];
      (runner as any).runAnalyzer = async (_item: any, _bytes: Buffer, _snapshot: any, feedback: string) => {
        analyzerFeedback.push(feedback);
        return {
          mode: "api",
          parsed: {},
          evidence: { type: "http", requests: [] },
          flags: [],
          result: {
            recommended_instruction: "Send the API request.",
            recommended_groups: [
              {
                id: "group_1",
                title: "Send request",
                intent: "Send request",
                callIds: [],
                apiCalls: [],
              },
            ],
            recommended_rubrics: [
              {
                name: "request_sent",
                description: "The expected API request is sent.",
                category: "process",
                checker_key: "request_sent",
                scoring: { 0: "Request missing", 1: "Request present" },
              },
            ],
          },
        };
      };
      const judgeResults = [
        {
          total_score: 7,
          max_score: 12,
          has_zeros: true,
          verdict: "NEEDS_REVISION",
          rationale: "Trajectory asks for an unsupported final action.",
          summary: "Needs revision.",
          dimensions: {
            trajectory_task_alignment: {
              score: 0,
              explanation: "Unsupported action is still present.",
            },
          },
        },
        {
          total_score: 12,
          max_score: 12,
          has_zeros: false,
          verdict: "PASS",
          rationale: "All dimensions pass.",
          summary: "Passes.",
          dimensions: {
            trajectory_task_alignment: {
              score: 2,
              explanation: "Unsupported action was removed.",
            },
          },
        },
      ];
      (runner as any).withLlmSlot = async () => judgeResults.shift();

      await (runner as any).runItem(item, job.id);

      const completed = store.getItem(item!.id);
      expect(completed?.status).toBe("completed");
      expect(analyzerFeedback).toHaveLength(2);
      expect(analyzerFeedback[0]).toBe("");
      expect(analyzerFeedback[1]).toContain("ITERATION MEMORY");
      expect(analyzerFeedback[1]).toContain("trajectory_task_alignment");
      expect(existsSync(resolve(dir, "exports", job.id, item!.id, "iteration-memory-iter-1.json"))).toBe(true);
      const evidence = JSON.parse(readFileSync(completed!.evidence_path!, "utf8"));
      expect(evidence.iterationMemory.updatedThroughIteration).toBe(2);
      expect(evidence.iterationMemory.resolvedDimensions).toContain("trajectory_task_alignment");
      store.close();
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });
});