import { readFileSync } from "node:fs"; import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; import JSZip from "jszip"; import { describe, expect, it } from "vitest"; import { QueueRunner } from "../queue/runner.js"; import { SqliteStore } from "../queue/store.js"; const repoRoot = fileURLToPath(new URL("../../../../", import.meta.url)); function readRepoFile(path: string): string { return readFileSync(resolve(repoRoot, path), "utf8"); } describe("judge failure export behavior", () => { it("builds the task package zip before marking a judge-quality failure", () => { const runner = readRepoFile("apps/api/src/queue/runner.ts"); const zipExportIndex = runner.indexOf('stage = "zip_export"'); const judgeFailureIndex = runner.indexOf("if (!passed)"); expect(zipExportIndex).toBeGreaterThan(-1); expect(judgeFailureIndex).toBeGreaterThan(-1); expect(zipExportIndex).toBeLessThan(judgeFailureIndex); }); it("allows download-all to include failed items when an export zip exists", () => { const analysisRoute = readRepoFile("apps/api/src/routes/analysis.ts"); expect(analysisRoute).not.toContain('it.status === "completed" &&'); expect(analysisRoute).toContain("it.export_zip_path &&"); }); it("caps judge refinement at five rounds and exposes per-round scores", () => { const runner = readRepoFile("apps/api/src/queue/runner.ts"); const analysisRoute = readRepoFile("apps/api/src/routes/analysis.ts"); expect(runner).toContain("const MAX_JUDGE_ITERATIONS = 5"); expect(runner).toContain("judge-result-iter-${iter}.json"); expect(analysisRoute).toContain("readJudgeHistory"); expect(analysisRoute).toContain("judgeHistory"); }); it("keeps a downloadable export when Judge throws before returning a verdict", async () => { const dir = mkdtempSync(resolve(tmpdir(), "rl-auto-judge-http-fail-")); try { const uploadPath = resolve(dir, "recording.zip"); const original = new JSZip(); original.file("network.json", JSON.stringify([{ method: "POST", url: "https://example.test/api" }])); writeFileSync(uploadPath, await original.generateAsync({ type: "nodebuffer" })); const store = new SqliteStore(resolve(dir, "tasks.db")); const job = store.createJob({ requestedMode: "api" }); const created = store.createItem({ jobId: job.id, ord: 1, filename: "recording.zip", uploadPath, detectedMode: "api", }); const item = store.takeNextQueuedItem(job.id); expect(item?.id).toBe(created.id); const logger = { child: () => logger, info: () => undefined, warn: () => undefined, error: () => undefined, } as any; const runner = new QueueRunner({ store, logger, config: { exportDir: resolve(dir, "exports"), itemMaxAttempts: 1, globalLlmConcurrency: 0, upstreamProxy: "", defaultApiKey: "unused", defaultBaseURL: "https://api.deepseek.com", defaultApiMode: "chat", defaultModel: "deepseek-v4-pro", } as any, }); (runner as any).runAnalyzer = async () => ({ mode: "api", parsed: {}, evidence: { type: "http", requests: [] }, flags: [], result: { recommended_instruction: "Send the API request.", recommended_groups: [ { id: "group_1", title: "Send request", intent: "Send request", callIds: [], apiCalls: [], }, ], recommended_rubrics: [ { name: "request_sent", description: "The expected API request is sent.", category: "process", checker_key: "request_sent", scoring: { 0: "Request missing", 1: "Request present" }, }, ], }, }); (runner as any).withLlmSlot = async () => { throw Object.assign( new Error("HTTP 403 - API key is not active or has been deleted"), { status: 403 }, ); }; await (runner as any).runItem(item, job.id); const failed = store.getItem(item!.id); expect(failed?.status).toBe("failed"); expect(failed?.last_stage).toBe("iteration_1:judge"); expect(failed?.export_zip_path).toBeTruthy(); expect(existsSync(failed!.export_zip_path!)).toBe(true); expect(failed?.task_package_path).toBeTruthy(); expect(existsSync(failed!.task_package_path!)).toBe(true); store.close(); } finally { rmSync(dir, { recursive: true, force: true }); } }); it("feeds optimized iteration memory into later analyzer rounds and persists it", async () => { const dir = mkdtempSync(resolve(tmpdir(), "rl-auto-iteration-memory-")); try { const uploadPath = resolve(dir, "recording.zip"); const original = new JSZip(); original.file("network.json", JSON.stringify([{ method: "POST", url: "https://example.test/api" }])); writeFileSync(uploadPath, await original.generateAsync({ type: "nodebuffer" })); const store = new SqliteStore(resolve(dir, "tasks.db")); const job = store.createJob({ requestedMode: "api" }); const created = store.createItem({ jobId: job.id, ord: 1, filename: "recording.zip", uploadPath, detectedMode: "api", }); const item = store.takeNextQueuedItem(job.id); expect(item?.id).toBe(created.id); const logger = { child: () => logger, info: () => undefined, warn: () => undefined, error: () => undefined, } as any; const runner = new QueueRunner({ store, logger, config: { exportDir: resolve(dir, "exports"), itemMaxAttempts: 1, globalLlmConcurrency: 0, upstreamProxy: "", defaultApiKey: "unused", defaultBaseURL: "https://api.deepseek.com", defaultApiMode: "chat", defaultModel: "deepseek-v4-pro", } as any, }); const analyzerFeedback: string[] = []; (runner as any).runAnalyzer = async (_item: any, _bytes: Buffer, _snapshot: any, feedback: string) => { analyzerFeedback.push(feedback); return { mode: "api", parsed: {}, evidence: { type: "http", requests: [] }, flags: [], result: { recommended_instruction: "Send the API request.", recommended_groups: [ { id: "group_1", title: "Send request", intent: "Send request", callIds: [], apiCalls: [], }, ], recommended_rubrics: [ { name: "request_sent", description: "The expected API request is sent.", category: "process", checker_key: "request_sent", scoring: { 0: "Request missing", 1: "Request present" }, }, ], }, }; }; const judgeResults = [ { total_score: 7, max_score: 12, has_zeros: true, verdict: "NEEDS_REVISION", rationale: "Trajectory asks for an unsupported final action.", summary: "Needs revision.", dimensions: { trajectory_task_alignment: { score: 0, explanation: "Unsupported action is still present.", }, }, }, { total_score: 12, max_score: 12, has_zeros: false, verdict: "PASS", rationale: "All dimensions pass.", summary: "Passes.", dimensions: { trajectory_task_alignment: { score: 2, explanation: "Unsupported action was removed.", }, }, }, ]; (runner as any).withLlmSlot = async () => judgeResults.shift(); await (runner as any).runItem(item, job.id); const completed = store.getItem(item!.id); expect(completed?.status).toBe("completed"); expect(analyzerFeedback).toHaveLength(2); expect(analyzerFeedback[0]).toBe(""); expect(analyzerFeedback[1]).toContain("ITERATION MEMORY"); expect(analyzerFeedback[1]).toContain("trajectory_task_alignment"); expect(existsSync(resolve(dir, "exports", job.id, item!.id, "iteration-memory-iter-1.json"))).toBe(true); const evidence = JSON.parse(readFileSync(completed!.evidence_path!, "utf8")); expect(evidence.iterationMemory.updatedThroughIteration).toBe(2); expect(evidence.iterationMemory.resolvedDimensions).toContain("trajectory_task_alignment"); store.close(); } finally { rmSync(dir, { recursive: true, force: true }); } }); });