tests/integration/combo-matrix/ordered.test.ts

// tests/integration/combo-matrix/ordered.test.ts
import test from "node:test";
import assert from "node:assert/strict";
import { createComboRoutingHarness } from "../_comboRoutingHarness.ts";

const h = await createComboRoutingHarness("combo-ordered");
const { BaseExecutor, combosDb, handleChat, buildRequest, seedConnection, resetStorage } = h;

function body(model: string) {
  return { model, stream: false, messages: [{ role: "user", content: `route ${model}` }] };
}

test.beforeEach(async () => {
  BaseExecutor.RETRY_CONFIG.delayMs = 0;
  await resetStorage();
});
test.afterEach(async () => {
  BaseExecutor.RETRY_CONFIG.delayMs = h.originalRetryDelayMs;
  await resetStorage();
});
test.after(async () => {
  await h.cleanup();
});

test("priority: always dispatches the first healthy target", async () => {
  await seedConnection("openai", { apiKey: "sk-openai-p" });
  await seedConnection("claude", { apiKey: "sk-claude-p" });
  await combosDb.createCombo({
    name: "m-priority",
    strategy: "priority",
    config: { maxRetries: 0, retryDelayMs: 0 },
    models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022"],
  });
  h.installRecordingFetch();

  for (let i = 0; i < 3; i++) {
    const r = await handleChat(buildRequest({ body: body("m-priority") }));
    assert.equal(r.status, 200);
  }
  assert.deepEqual(h.providersSeen(), ["openai", "openai", "openai"]);
});

test("priority: falls back to the next target when the first fails", async () => {
  await seedConnection("openai", { apiKey: "sk-openai-pf" });
  await seedConnection("claude", { apiKey: "sk-claude-pf" });
  await combosDb.createCombo({
    name: "m-priority-fail",
    strategy: "priority",
    config: { maxRetries: 0, retryDelayMs: 0 },
    models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022"],
  });
  // First upstream call (openai) fails → must fall over to claude.
  h.installRecordingFetch((call) => (call.index === 0 ? h.failure(503) : undefined));

  const r = await handleChat(buildRequest({ body: body("m-priority-fail") }));
  assert.equal(r.status, 200);
  assert.deepEqual(h.providersSeen(), ["openai", "claude"]);
});

test("fill-first: keeps using the first target until it fails, then moves on", async () => {
  await seedConnection("openai", { apiKey: "sk-openai-ff" });
  await seedConnection("claude", { apiKey: "sk-claude-ff" });
  await combosDb.createCombo({
    name: "m-fill-first",
    strategy: "fill-first",
    config: { maxRetries: 0, retryDelayMs: 0 },
    models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022"],
  });
  h.installRecordingFetch();

  // Healthy openai → all requests hit openai (fill-first preserves priority order).
  for (let i = 0; i < 3; i++) {
    const r = await handleChat(buildRequest({ body: body("m-fill-first") }));
    assert.equal(r.status, 200);
  }
  assert.deepEqual(h.providersSeen(), ["openai", "openai", "openai"]);
});

test("round-robin: cycles through targets in batches (sticky limit = 3 default)", async () => {
  await seedConnection("openai", { apiKey: "sk-openai-rr" });
  await seedConnection("claude", { apiKey: "sk-claude-rr" });
  await seedConnection("gemini", { apiKey: "sk-gemini-rr" });
  await combosDb.createCombo({
    name: "m-rr",
    strategy: "round-robin",
    config: { maxRetries: 0, retryDelayMs: 0 },
    models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022", "gemini/gemini-2.5-flash"],
  });
  h.installRecordingFetch();

  for (let i = 0; i < 9; i++) {
    const r = await handleChat(buildRequest({ body: body("m-rr") }));
    assert.equal(r.status, 200);
  }
  assert.deepEqual(h.providersSeen(), [
    "openai", "openai", "openai",
    "claude", "claude", "claude",
    "gemini", "gemini", "gemini",
  ]);
});

test("least-used: prefers the target with the fewest recent uses", async () => {
  await seedConnection("openai", { apiKey: "sk-openai-lu" });
  await seedConnection("claude", { apiKey: "sk-claude-lu" });
  await combosDb.createCombo({
    name: "m-least-used",
    strategy: "least-used",
    config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
    models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022"],
  });
  h.installRecordingFetch();

  // Over an even number of calls each target should be picked roughly equally;
  // least-used must not pin to a single provider.
  for (let i = 0; i < 6; i++) {
    const r = await handleChat(buildRequest({ body: body("m-least-used") }));
    assert.equal(r.status, 200);
  }
  const seen = h.providersSeen();
  assert.ok(seen.includes("openai"), "least-used must reach openai");
  assert.ok(seen.includes("claude"), "least-used must reach claude");
});

test("random: over many calls reaches every target", async () => {
  await seedConnection("openai", { apiKey: "sk-openai-rnd" });
  await seedConnection("claude", { apiKey: "sk-claude-rnd" });
  await seedConnection("gemini", { apiKey: "sk-gemini-rnd" });
  await combosDb.createCombo({
    name: "m-random",
    strategy: "random",
    config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
    models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022", "gemini/gemini-2.5-flash"],
  });
  h.installRecordingFetch();

  for (let i = 0; i < 30; i++) {
    const r = await handleChat(buildRequest({ body: body("m-random") }));
    assert.equal(r.status, 200);
  }
  const unique = new Set(h.providersSeen());
  assert.equal(unique.size, 3, "random must reach all three providers over 30 calls");
});

test("strict-random: uses every target once before repeating (deck, no early repeats)", async () => {
  const { _resetAllDecks } = await import("../../../src/shared/utils/shuffleDeck.ts");
  _resetAllDecks();
  await seedConnection("openai", { apiKey: "sk-openai-sr" });
  await seedConnection("claude", { apiKey: "sk-claude-sr" });
  await seedConnection("gemini", { apiKey: "sk-gemini-sr" });
  await combosDb.createCombo({
    name: "m-strict-random",
    strategy: "strict-random",
    config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
    models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022", "gemini/gemini-2.5-flash"],
  });
  h.installRecordingFetch();

  for (let i = 0; i < 3; i++) {
    const r = await handleChat(buildRequest({ body: body("m-strict-random") }));
    assert.equal(r.status, 200);
  }
  // First full deck cycle: each provider exactly once (order varies, set is full).
  assert.deepEqual(new Set(h.providersSeen()), new Set(["openai", "claude", "gemini"]));
});

test("p2c: spreads load across targets over many calls (no single-target pin)", async () => {
  await seedConnection("openai", { apiKey: "sk-openai-p2c" });
  await seedConnection("claude", { apiKey: "sk-claude-p2c" });
  await seedConnection("gemini", { apiKey: "sk-gemini-p2c" });
  await combosDb.createCombo({
    name: "m-p2c",
    strategy: "p2c",
    config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
    models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022", "gemini/gemini-2.5-flash"],
  });
  h.installRecordingFetch();

  for (let i = 0; i < 30; i++) {
    const r = await handleChat(buildRequest({ body: body("m-p2c") }));
    assert.equal(r.status, 200);
  }
  assert.ok(new Set(h.providersSeen()).size >= 2, "p2c must spread across at least two targets");
});