artifacts/api-server/src/lib/capability/inner-loop/__tests__/runner.test.ts

/**
 * runInnerLoop — 8 case 覆盖 8 步 ReAct loop 装配。
 *
 * 每个 case 用 hook 拦截持久化(避免触 DB),只 assert 行为。
 *   1. happy path: 8 步全过
 *   2. skipSteps: code 步被 skip → 仍 outcome=completed
 *   3. step 未声明 → status=skipped
 *   4. LLM 错 + fallback → status=fallback,used_llm=false
 *   5. LLM 错 + 无 fallback → outcome=aborted
 *   6. non-LLM 错 → outcome=failed
 *   7. usedLlmPerStep 反映真实(混合 LLM + system)
 *   8. step events 顺序 + payload_hash 写入
 */
import { test } from "node:test";
import assert from "node:assert/strict";

process.env.DATABASE_URL ??= "postgres://test:test@127.0.0.1:5432/test";

import {
  runInnerLoop,
  _setInnerLoopPersistenceHooks,
  INNER_LOOP_STEPS,
  type StepDecl,
  type StepEvent,
} from "../runner.ts";
import { LlmRoleDisabled, LlmRoleResponseInvalid } from "../../../llm-roles/errors.ts";

function captureHooks() {
  const stepEvents: StepEvent[] = [];
  let runOutcome: string | null = null;
  let runUsed: Record<string, boolean> | null = null;
  _setInnerLoopPersistenceHooks({
    step: (e) => stepEvents.push(e),
    runStart: () => {},
    runEnd: (_id, outcome, _steps, used) => {
      runOutcome = outcome;
      runUsed = used;
    },
  });
  return {
    stepEvents,
    get runOutcome() {
      return runOutcome;
    },
    get runUsed() {
      return runUsed;
    },
  };
}

function reset() {
  _setInnerLoopPersistenceHooks({ step: null, runStart: null, runEnd: null });
}

function passthrough(name: string, role: string | null = null, usesLlm = false): StepDecl {
  return {
    name: name as StepDecl["name"],
    usesLlm,
    role,
    fn: async () => ({ ok: true, step: name }),
  };
}

test("runInnerLoop — happy path: 8 steps all pass, outcome=completed", async () => {
  const cap = captureHooks();
  try {
    const steps: StepDecl[] = INNER_LOOP_STEPS.map((s) => passthrough(s));
    const r = await runInnerLoop({ capabilityId: "cap_a", steps });
    assert.equal(r.outcome, "completed");
    assert.equal(r.steps.length, 8);
    assert.ok(r.steps.every((e) => e.status === "ok"));
    assert.equal(cap.runOutcome, "completed");
  } finally {
    reset();
  }
});

test("runInnerLoop — skipSteps: code is marked skipped, outcome still completed", async () => {
  const cap = captureHooks();
  try {
    const steps: StepDecl[] = INNER_LOOP_STEPS.map((s) => {
      const d = passthrough(s);
      if (s === "code") d.skip = true;
      return d;
    });
    const r = await runInnerLoop({ capabilityId: "cap_a", steps });
    assert.equal(r.outcome, "completed");
    const code = r.steps.find((e) => e.stepName === "code");
    assert.ok(code);
    assert.equal(code!.status, "skipped");
    assert.equal(cap.runOutcome, "completed");
  } finally {
    reset();
  }
});

test("runInnerLoop — step not declared in manifest is recorded as skipped", async () => {
  captureHooks();
  try {
    const steps: StepDecl[] = INNER_LOOP_STEPS.filter((s) => s !== "search").map(
      (s) => passthrough(s),
    );
    const r = await runInnerLoop({ capabilityId: "cap_a", steps });
    const search = r.steps.find((e) => e.stepName === "search");
    assert.ok(search);
    assert.equal(search!.status, "skipped");
    assert.equal(search!.errorMessage, "step not declared in manifest");
    assert.equal(r.outcome, "completed");
  } finally {
    reset();
  }
});

test("runInnerLoop — LLM error + fallback: status=fallback, used_llm=false", async () => {
  captureHooks();
  try {
    const steps: StepDecl[] = INNER_LOOP_STEPS.map((s) => {
      if (s === "diagnose") {
        return {
          name: "diagnose",
          usesLlm: true,
          role: "diagnoser",
          fn: async () => {
            throw new LlmRoleDisabled();
          },
          fallback: async () => ({ heuristic: "ok" }),
        };
      }
      return passthrough(s);
    });
    const r = await runInnerLoop({ capabilityId: "cap_a", steps });
    const diag = r.steps.find((e) => e.stepName === "diagnose");
    assert.ok(diag);
    assert.equal(diag!.status, "fallback");
    assert.equal(r.usedLlmPerStep.diagnose, false);
    assert.equal(r.outcome, "completed");
  } finally {
    reset();
  }
});

test("runInnerLoop — LLM error + no fallback: outcome=aborted", async () => {
  captureHooks();
  try {
    const steps: StepDecl[] = INNER_LOOP_STEPS.map((s) => {
      if (s === "judge") {
        return {
          name: "judge",
          usesLlm: true,
          role: "judge",
          fn: async () => {
            throw new LlmRoleResponseInvalid("judge", null, "");
          },
          // no fallback
        };
      }
      return passthrough(s);
    });
    const r = await runInnerLoop({ capabilityId: "cap_a", steps });
    assert.equal(r.outcome, "aborted");
    const judge = r.steps.find((e) => e.stepName === "judge");
    assert.equal(judge!.status, "error");
  } finally {
    reset();
  }
});

test("runInnerLoop — non-LLM error: outcome=failed (no fallback attempted)", async () => {
  captureHooks();
  try {
    let fallbackCalled = false;
    const steps: StepDecl[] = INNER_LOOP_STEPS.map((s) => {
      if (s === "act") {
        return {
          name: "act",
          usesLlm: false,
          role: null,
          fn: async () => {
            throw new Error("system blew up");
          },
          fallback: async () => {
            fallbackCalled = true;
            return {};
          },
        };
      }
      return passthrough(s);
    });
    const r = await runInnerLoop({ capabilityId: "cap_a", steps });
    assert.equal(r.outcome, "failed");
    assert.equal(fallbackCalled, false, "fallback should not run for non-LLM errors");
  } finally {
    reset();
  }
});

test("runInnerLoop — usedLlmPerStep mirrors mixed LLM+system declarations", async () => {
  captureHooks();
  try {
    const llmSteps = new Set(["diagnose", "plan", "judge"]);
    const steps: StepDecl[] = INNER_LOOP_STEPS.map((s) =>
      passthrough(s, llmSteps.has(s) ? s : null, llmSteps.has(s)),
    );
    const r = await runInnerLoop({ capabilityId: "cap_a", steps });
    assert.equal(r.usedLlmPerStep.diagnose, true);
    assert.equal(r.usedLlmPerStep.plan, true);
    assert.equal(r.usedLlmPerStep.judge, true);
    assert.equal(r.usedLlmPerStep.observe, false);
    assert.equal(r.usedLlmPerStep.code, false);
  } finally {
    reset();
  }
});

test("runInnerLoop — step events have stable order and payload_hash", async () => {
  const cap = captureHooks();
  try {
    const steps: StepDecl[] = INNER_LOOP_STEPS.map((s) => passthrough(s));
    const r = await runInnerLoop({ capabilityId: "cap_a", steps });
    // step_index strictly ascending, matches INNER_LOOP_STEPS order
    for (let i = 0; i < r.steps.length; i++) {
      assert.equal(r.steps[i]!.stepIndex, i);
      assert.equal(r.steps[i]!.stepName, INNER_LOOP_STEPS[i]);
      assert.ok(r.steps[i]!.payloadHash, "ok step should have payload_hash");
    }
    // Hook receives them in the same order
    for (let i = 0; i < cap.stepEvents.length; i++) {
      assert.equal(cap.stepEvents[i]!.stepIndex, i);
    }
  } finally {
    reset();
  }
});