worldmonitor/convex/__tests__/intelHistory.test.ts

import { convexTest } from "convex-test";
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
import { internal } from "../_generated/api";
import schema from "../schema";
import {
  INTEL_HISTORY_EMBED_DIMS,
  INTEL_HISTORY_MAX_APPEND_RECORDS,
  INTEL_HISTORY_MAX_RETRACT_IDENTIFIERS,
  INTEL_HISTORY_RETENTION_DAYS,
  TIMELINE_MAX_LIMIT,
} from "../intelHistory";

const modules = import.meta.glob("../**/*.ts");

const RELAY_SECRET = "test-relay-secret-intel-history-8f2a91";
const CONVEX_SECRET = "test-convex-secret-intel-history-91bd42";
const NOW = 1_780_000_000_000;
const DAY = 86_400_000;
const DIMS = INTEL_HISTORY_EMBED_DIMS;

/**
 * Unit vector along `axis`, optionally tilted toward the next axis. Cosine
 * similarity against `unitVector(0)` is then exactly 1 for axis 0, 0 for any
 * other pure axis, and 1/sqrt(1+tilt^2) for a tilted axis-0 vector — so score
 * ordering in the vector-search tests is arithmetic, not luck.
 */
function unitVector(axis: number, tilt = 0): number[] {
  const vec = new Array<number>(DIMS).fill(0);
  vec[axis] = 1;
  if (tilt !== 0) vec[(axis + 1) % DIMS] = tilt;
  return vec;
}

type AppendRecord = {
  dedupeKey: string;
  country?: string;
  category?: string;
  title: string;
  summary?: string;
  sourceUrl?: string;
  occurredAt: number;
  embedding: number[];
};

function record(overrides: Partial<AppendRecord> = {}): AppendRecord {
  return {
    dedupeKey: "dk-1",
    title: "Shelling reported near Kharkiv",
    occurredAt: NOW - DAY,
    embedding: unitVector(0),
    ...overrides,
  };
}

function appendArgs(
  records: AppendRecord[],
  overrides: { domain?: string; resource?: string; runId?: string } = {},
) {
  return {
    domain: "conflict",
    resource: "conflict-events",
    runId: "run-1",
    records,
    ...overrides,
  };
}

/** Insert straight to the table, bypassing the mutation, for read-path setup. */
async function seed(
  t: ReturnType<typeof convexTest>,
  rows: Array<Record<string, unknown>>,
) {
  await t.run(async (ctx) => {
    for (const row of rows) {
      await ctx.db.insert("intelHistory", {
        domain: "conflict",
        resource: "conflict-events",
        title: "seeded",
        ingestedAt: NOW,
        runId: "run-seed",
        embedding: unitVector(0),
        ...row,
      } as never);
    }
  });
}

/** Append intentionally fails closed unless deploy-style lock seeding ran. */
async function intelHistoryAppendTest() {
  const t = convexTest(schema, modules);
  await t.mutation(internal.intelHistory._seedAppendLock, {});
  return t;
}

describe("intelHistory.append", () => {
  beforeEach(() => {
    vi.useFakeTimers();
    vi.setSystemTime(NOW);
  });
  afterEach(() => vi.useRealTimers());

  test("inserts records stamped with the run-level fields and ingestedAt", async () => {
    const t = await intelHistoryAppendTest();

    const res = await t.mutation(
      internal.intelHistory.append,
      appendArgs([
        record({ dedupeKey: "a", country: "UA", category: "battle" }),
        record({
          dedupeKey: "b",
          title: "Drone strike",
          summary: "Overnight drone strike on infrastructure",
          sourceUrl: "https://example.test/1",
          occurredAt: NOW - 2 * DAY,
          embedding: unitVector(1),
        }),
      ]),
    );

    expect(res).toEqual({ inserted: 2, skipped: 0, retracted: 0 });

    const rows = await t.run((ctx) => ctx.db.query("intelHistory").collect());
    expect(rows).toHaveLength(2);
    const byKey = new Map(rows.map((r) => [r.dedupeKey, r]));
    expect(byKey.get("a")).toMatchObject({
      domain: "conflict",
      resource: "conflict-events",
      runId: "run-1",
      country: "UA",
      category: "battle",
      ingestedAt: NOW,
      occurredAt: NOW - DAY,
    });
    expect(byKey.get("a")!.embedding).toHaveLength(DIMS);
    expect(byKey.get("b")).toMatchObject({
      summary: "Overnight drone strike on infrastructure",
      sourceUrl: "https://example.test/1",
    });
  });

  test("skips a dedupeKey that already exists from an earlier run", async () => {
    const t = await intelHistoryAppendTest();

    await t.mutation(
      internal.intelHistory.append,
      appendArgs([record({ dedupeKey: "dup" })]),
    );
    const res = await t.mutation(
      internal.intelHistory.append,
      appendArgs([
        record({ dedupeKey: "dup", title: "Rewritten headline" }),
        record({ dedupeKey: "fresh" }),
      ]),
      );

    expect(res).toEqual({ inserted: 1, skipped: 1, retracted: 0 });
    const rows = await t.run((ctx) => ctx.db.query("intelHistory").collect());
    expect(rows).toHaveLength(2);
    // The pre-existing row is untouched — append never rewrites history.
    expect(rows.find((r) => r.dedupeKey === "dup")!.title).toBe(
      "Shelling reported near Kharkiv",
    );
  });

  test("dedupes repeated keys inside a single batch", async () => {
    const t = await intelHistoryAppendTest();

    const res = await t.mutation(
      internal.intelHistory.append,
      appendArgs([record({ dedupeKey: "x" }), record({ dedupeKey: "x" })]),
    );

    expect(res).toEqual({ inserted: 1, skipped: 1, retracted: 0 });
  });

  test("serializes simultaneous first-seen appends through the seeded lock", async () => {
    const t = await intelHistoryAppendTest();

    const results = await Promise.all([
      t.mutation(
        internal.intelHistory.append,
        appendArgs([record({ dedupeKey: "simultaneous" })], { runId: "run-a" }),
      ),
      t.mutation(
        internal.intelHistory.append,
        appendArgs([record({ dedupeKey: "simultaneous" })], { runId: "run-b" }),
      ),
    ]);

    expect(results).toContainEqual({ inserted: 1, skipped: 0, retracted: 0 });
    expect(results).toContainEqual({ inserted: 0, skipped: 1, retracted: 0 });
    const rows = await t.run((ctx) =>
      ctx.db
        .query("intelHistory")
        .withIndex("by_dedupeKey", (q) => q.eq("dedupeKey", "simultaneous"))
        .collect(),
    );
    expect(rows).toHaveLength(1);
  });

  test("rejects a batch larger than the append cap", async () => {
    const t = await intelHistoryAppendTest();
    const oversized = Array.from(
      { length: INTEL_HISTORY_MAX_APPEND_RECORDS + 1 },
      (_, i) => record({ dedupeKey: `k-${i}` }),
    );

    await expect(
      t.mutation(internal.intelHistory.append, appendArgs(oversized)),
    ).rejects.toThrow(/at most 100 records/i);

    const rows = await t.run((ctx) => ctx.db.query("intelHistory").collect());
    expect(rows).toHaveLength(0);
  });

  test("rejects a record whose embedding is not 512-dimensional", async () => {
    const t = await intelHistoryAppendTest();

    await expect(
      t.mutation(
        internal.intelHistory.append,
        appendArgs([
          record({ dedupeKey: "ok" }),
          record({ dedupeKey: "bad", embedding: new Array(256).fill(0.1) }),
        ]),
      ),
    ).rejects.toThrow(/embedding/i);

    // Validation runs before any insert, so the whole batch is rejected.
    const rows = await t.run((ctx) => ctx.db.query("intelHistory").collect());
    expect(rows).toHaveLength(0);
  });

  test("rejects a non-finite embedding component", async () => {
    const t = await intelHistoryAppendTest();
    const poisoned = unitVector(0);
    poisoned[3] = Number.NaN;

    await expect(
      t.mutation(
        internal.intelHistory.append,
        appendArgs([record({ dedupeKey: "nan", embedding: poisoned })]),
      ),
    ).rejects.toThrow(/finite/i);
  });

  test("fails closed when the deploy-seeded append lock is absent", async () => {
    const t = convexTest(schema, modules);

    await expect(
      t.mutation(internal.intelHistory.append, appendArgs([record()])),
    ).rejects.toThrow(/APPEND_LOCK_NOT_SEEDED/);

    const rows = await t.run((ctx) => ctx.db.query("intelHistory").collect());
    expect(rows).toHaveLength(0);
  });

  test("idempotently seeds the document-backed append lock", async () => {
    const t = convexTest(schema, modules);

    expect(await t.mutation(internal.intelHistory._seedAppendLock, {})).toEqual({
      seeded: 1,
    });
    expect(await t.mutation(internal.intelHistory._seedAppendLock, {})).toEqual({
      seeded: 0,
    });
    const locks = await t.run((ctx) =>
      ctx.db.query("intelHistoryAppendLocks").collect(),
    );
    expect(locks).toHaveLength(1);
  });
});

describe("intelHistory.timeline", () => {
  test("returns the domain's events newest-first within the occurredAt range", async () => {
    const t = convexTest(schema, modules);
    await seed(t, [
      { dedupeKey: "old", occurredAt: NOW - 10 * DAY, title: "old" },
      { dedupeKey: "mid", occurredAt: NOW - 5 * DAY, title: "mid" },
      { dedupeKey: "new", occurredAt: NOW - DAY, title: "new" },
      {
        dedupeKey: "other-domain",
        domain: "energy",
        occurredAt: NOW - 2 * DAY,
        title: "energy",
      },
    ]);

    const res = await t.query(internal.intelHistory.timeline, {
      domain: "conflict",
      from: NOW - 6 * DAY,
      to: NOW,
    });

    expect(res.records.map((r) => r.title)).toEqual(["new", "mid"]);
  });

  test("post-filters country when the domain index is used", async () => {
    const t = convexTest(schema, modules);
    await seed(t, [
      { dedupeKey: "ua", country: "UA", occurredAt: NOW - DAY, title: "ua" },
      { dedupeKey: "sd", country: "SD", occurredAt: NOW - 2 * DAY, title: "sd" },
      { dedupeKey: "none", occurredAt: NOW - 3 * DAY, title: "none" },
    ]);

    const res = await t.query(internal.intelHistory.timeline, {
      domain: "conflict",
      country: "UA",
    });

    expect(res.records.map((r) => r.title)).toEqual(["ua"]);
    expect(res.partial).toBe(false);
  });

  test("marks a full post-filter candidate window as partial instead of false-empty", async () => {
    const t = convexTest(schema, modules);
    await seed(t, [
      { dedupeKey: "sd-1", country: "SD", occurredAt: NOW - DAY, title: "sd-1" },
      { dedupeKey: "sd-2", country: "SD", occurredAt: NOW - 2 * DAY, title: "sd-2" },
      { dedupeKey: "sd-3", country: "SD", occurredAt: NOW - 3 * DAY, title: "sd-3" },
      { dedupeKey: "sd-4", country: "SD", occurredAt: NOW - 4 * DAY, title: "sd-4" },
      { dedupeKey: "ua-after-window", country: "UA", occurredAt: NOW - 5 * DAY, title: "ua" },
    ]);

    const res = await t.query(internal.intelHistory.timeline, {
      domain: "conflict",
      country: "UA",
      limit: 1,
    });

    expect(res.records).toEqual([]);
    expect(res.partial).toBe(true);
  });

  test("serves a country-only query off the country index across domains", async () => {
    const t = convexTest(schema, modules);
    await seed(t, [
      { dedupeKey: "c1", country: "UA", occurredAt: NOW - DAY, title: "conflict-ua" },
      {
        dedupeKey: "e1",
        domain: "energy",
        country: "UA",
        occurredAt: NOW - 2 * DAY,
        title: "energy-ua",
      },
      { dedupeKey: "c2", country: "SD", occurredAt: NOW - 3 * DAY, title: "conflict-sd" },
    ]);

    const res = await t.query(internal.intelHistory.timeline, { country: "UA" });

    expect(res.records.map((r) => r.title)).toEqual(["conflict-ua", "energy-ua"]);
  });

  test("projects a stable id, drops the embedding, and omits absent optionals", async () => {
    const t = convexTest(schema, modules);
    await seed(t, [
      {
        dedupeKey: "p1",
        occurredAt: NOW - DAY,
        title: "projected",
        summary: "a summary",
        sourceUrl: "https://example.test/p1",
        country: "UA",
        category: "battle",
      },
      { dedupeKey: "p2", occurredAt: NOW - 2 * DAY, title: "bare" },
    ]);

    const res = await t.query(internal.intelHistory.timeline, { domain: "conflict" });

    const [full, bare] = res.records;
    expect(full).toEqual({
      id: expect.any(String),
      domain: "conflict",
      resource: "conflict-events",
      country: "UA",
      category: "battle",
      title: "projected",
      summary: "a summary",
      sourceUrl: "https://example.test/p1",
      occurredAt: NOW - DAY,
      ingestedAt: NOW,
      runId: "run-seed",
      dedupeKey: "p1",
    });
    expect(full).not.toHaveProperty("embedding");
    expect(full).not.toHaveProperty("_id");
    // Absent optionals are stripped by Convex value serialization, not echoed
    // back as explicit nulls.
    expect(bare).not.toHaveProperty("country");
    expect(bare).not.toHaveProperty("summary");
  });

  test("honours an explicit limit and clamps it to the documented maximum", async () => {
    const t = convexTest(schema, modules);
    const shared = unitVector(0);
    await seed(
      t,
      Array.from({ length: TIMELINE_MAX_LIMIT + 1 }, (_, i) => ({
        dedupeKey: `bulk-${i}`,
        occurredAt: NOW - i * 1000,
        title: `bulk-${i}`,
        embedding: shared,
      })),
    );

    const two = await t.query(internal.intelHistory.timeline, {
      domain: "conflict",
      limit: 2,
    });
    expect(two.records.map((r) => r.title)).toEqual(["bulk-0", "bulk-1"]);

    const clamped = await t.query(internal.intelHistory.timeline, {
      domain: "conflict",
      limit: 10_000,
    });
    expect(clamped.records).toHaveLength(TIMELINE_MAX_LIMIT);
  });

  test("throws when neither domain nor country scopes the read", async () => {
    const t = convexTest(schema, modules);
    await expect(
      t.query(internal.intelHistory.timeline, { from: NOW - DAY }),
    ).rejects.toThrow(/domain.*country|country.*domain/i);
  });
});

describe("intelHistory.getByIds", () => {
  test("preserves input order, skips missing ids, and drops the embedding", async () => {
    const t = convexTest(schema, modules);
    await seed(t, [
      { dedupeKey: "g1", occurredAt: NOW - DAY, title: "first" },
      { dedupeKey: "g2", occurredAt: NOW - 2 * DAY, title: "second" },
    ]);

    const rows = await t.run((ctx) => ctx.db.query("intelHistory").collect());
    const first = rows.find((r) => r.dedupeKey === "g1")!;
    const second = rows.find((r) => r.dedupeKey === "g2")!;

    // Delete one row so its id is dangling when we hydrate.
    const ghostId = await t.run(async (ctx) => {
      const id = await ctx.db.insert("intelHistory", {
        domain: "conflict",
        resource: "conflict-events",
        title: "ghost",
        occurredAt: NOW,
        ingestedAt: NOW,
        runId: "run-seed",
        dedupeKey: "ghost",
        embedding: unitVector(0),
      });
      await ctx.db.delete(id);
      return id;
    });

    const res = await t.query(internal.intelHistory.getByIds, {
      ids: [second._id, ghostId, first._id],
    });

    expect(res.map((r) => r.title)).toEqual(["second", "first"]);
    expect(res[0]).not.toHaveProperty("embedding");
    expect(res[0].id).toBe(second._id);
  });
});

describe("intelHistory.search", () => {
  test("ranks by cosine similarity and attaches _score", async () => {
    const t = convexTest(schema, modules);
    await seed(t, [
      { dedupeKey: "exact", title: "exact", occurredAt: NOW - DAY, embedding: unitVector(0) },
      { dedupeKey: "near", title: "near", occurredAt: NOW - DAY, embedding: unitVector(0, 0.5) },
      { dedupeKey: "far", title: "far", occurredAt: NOW - DAY, embedding: unitVector(7) },
    ]);

    const res = await t.action(internal.intelHistory.search, {
      embedding: unitVector(0),
    });

    expect(res.records.map((r) => r.title)).toEqual(["exact", "near", "far"]);
    expect(res.records[0]._score).toBeCloseTo(1, 6);
    expect(res.records[1]._score).toBeCloseTo(1 / Math.sqrt(1.25), 6);
    expect(res.records[2]._score).toBeCloseTo(0, 6);
    expect(res.records[0]).not.toHaveProperty("embedding");
  });

  test("optionally excludes zero and negative score matches without changing the default search", async () => {
    const t = convexTest(schema, modules);
    const opposite = unitVector(0).map((component) => -component);
    await seed(t, [
      { dedupeKey: "exact", title: "exact", occurredAt: NOW - DAY, embedding: unitVector(0) },
      { dedupeKey: "zero", title: "zero", occurredAt: NOW - DAY, embedding: unitVector(7) },
      { dedupeKey: "negative", title: "negative", occurredAt: NOW - DAY, embedding: opposite },
    ]);

    const res = await t.action(internal.intelHistory.search, {
      embedding: unitVector(0),
      limit: 3,
      minScore: 0.1,
    });

    expect(res.records.map((r) => r.title)).toEqual(["exact"]);
  });

  test("pushes the domain filter into the vector index", async () => {
    const t = convexTest(schema, modules);
    await seed(t, [
      { dedupeKey: "c", title: "conflict", occurredAt: NOW - DAY, embedding: unitVector(0) },
      {
        dedupeKey: "e",
        domain: "energy",
        title: "energy",
        occurredAt: NOW - DAY,
        embedding: unitVector(0),
      },
    ]);

    const res = await t.action(internal.intelHistory.search, {
      embedding: unitVector(0),
      domain: "energy",
    });

    expect(res.records.map((r) => r.title)).toEqual(["energy"]);
  });

  test("post-filters country and the occurredAt range", async () => {
    const t = convexTest(schema, modules);
    await seed(t, [
      {
        dedupeKey: "hit",
        country: "UA",
        title: "hit",
        occurredAt: NOW - DAY,
        embedding: unitVector(0),
      },
      {
        dedupeKey: "wrong-country",
        country: "SD",
        title: "wrong-country",
        occurredAt: NOW - DAY,
        embedding: unitVector(0),
      },
      {
        dedupeKey: "too-old",
        country: "UA",
        title: "too-old",
        occurredAt: NOW - 90 * DAY,
        embedding: unitVector(0),
      },
    ]);

    const res = await t.action(internal.intelHistory.search, {
      embedding: unitVector(0),
      domain: "conflict",
      country: "UA",
      from: NOW - 7 * DAY,
      to: NOW,
    });

    expect(res.records.map((r) => r.title)).toEqual(["hit"]);
    expect(res.partial).toBe(false);
  });

  test("marks a full vector candidate window as partial instead of false-empty", async () => {
    const t = convexTest(schema, modules);
    await seed(t, [
      { dedupeKey: "sd-1", country: "SD", title: "sd-1", occurredAt: NOW - DAY, embedding: unitVector(0) },
      { dedupeKey: "sd-2", country: "SD", title: "sd-2", occurredAt: NOW - DAY, embedding: unitVector(0) },
      { dedupeKey: "sd-3", country: "SD", title: "sd-3", occurredAt: NOW - DAY, embedding: unitVector(0) },
      { dedupeKey: "sd-4", country: "SD", title: "sd-4", occurredAt: NOW - DAY, embedding: unitVector(0) },
      { dedupeKey: "ua-after-window", country: "UA", title: "ua", occurredAt: NOW - DAY, embedding: unitVector(0, 0.5) },
    ]);

    const res = await t.action(internal.intelHistory.search, {
      embedding: unitVector(0),
      domain: "conflict",
      country: "UA",
      limit: 1,
    });

    expect(res.records).toEqual([]);
    expect(res.partial).toBe(true);
  });

  test("clamps limit and rejects a wrong-dimension query vector", async () => {
    const t = convexTest(schema, modules);
    await seed(t, [
      { dedupeKey: "s1", title: "s1", occurredAt: NOW - DAY, embedding: unitVector(0) },
      { dedupeKey: "s2", title: "s2", occurredAt: NOW - DAY, embedding: unitVector(0, 0.5) },
    ]);

    const limited = await t.action(internal.intelHistory.search, {
      embedding: unitVector(0),
      limit: 1,
    });
    expect(limited.records).toHaveLength(1);

    await expect(
      t.action(internal.intelHistory.search, { embedding: [0.1, 0.2] }),
    ).rejects.toThrow(/embedding/i);
  });
});

describe("intelHistory.prune", () => {
  beforeEach(() => vi.useFakeTimers());
  afterEach(() => {
    vi.clearAllTimers();
    vi.useRealTimers();
  });

  test("ages rows out by ingestedAt, so a backfill of old events survives", async () => {
    const t = convexTest(schema, modules);
    const beyond = NOW - (INTEL_HISTORY_RETENTION_DAYS + 1) * DAY;
    await seed(t, [
      { dedupeKey: "aged", ingestedAt: beyond, occurredAt: beyond, title: "aged" },
      { dedupeKey: "recent", ingestedAt: NOW - DAY, occurredAt: NOW - DAY, title: "recent" },
      // Backfilled yesterday, but the event itself predates the window. This
      // row is the whole reason retention keys on ingestedAt — pruning by
      // occurredAt would delete a backfill the night it landed.
      {
        dedupeKey: "backfilled",
        ingestedAt: NOW - DAY,
        occurredAt: beyond,
        title: "backfilled",
      },
    ]);

    const res = await t.mutation(internal.intelHistory.prune, { now: NOW });

    expect(res).toEqual({ deleted: 1, deletedRetractions: 0, rescheduled: false });
    const rows = await t.run((ctx) => ctx.db.query("intelHistory").collect());
    expect(rows.map((r) => r.dedupeKey).sort()).toEqual(["backfilled", "recent"]);
  });

  test("self-reschedules until the backlog drains", async () => {
    const t = convexTest(schema, modules);
    const beyond = NOW - (INTEL_HISTORY_RETENTION_DAYS + 1) * DAY;
    await seed(
      t,
      Array.from({ length: 3 }, (_, i) => ({
        dedupeKey: `aged-${i}`,
        ingestedAt: beyond,
        occurredAt: beyond,
        title: `aged-${i}`,
      })),
    );

    const first = await t.mutation(internal.intelHistory.prune, {
      now: NOW,
      limit: 2,
    });
    expect(first).toEqual({ deleted: 2, deletedRetractions: 0, rescheduled: true });

    await t.finishAllScheduledFunctions(vi.runAllTimers);

    const rows = await t.run((ctx) => ctx.db.query("intelHistory").collect());
    expect(rows).toHaveLength(0);
  });

  // A batch of 0 makes take(0) return [], and an unguarded `length >= batch`
  // reads 0 >= 0 as a full batch — rescheduling forever while deleting
  // nothing. The cron passes {}, but prune is operator-callable.
  test("does not reschedule itself forever when called with limit 0", async () => {
    const t = convexTest(schema, modules);
    const beyond = NOW - (INTEL_HISTORY_RETENTION_DAYS + 1) * DAY;
    await seed(t, [
      { dedupeKey: "aged-0", ingestedAt: beyond, occurredAt: beyond, title: "aged-0" },
    ]);

    const result = await t.mutation(internal.intelHistory.prune, { now: NOW, limit: 0 });

    // Floors to 1: it makes progress instead of spinning on an empty batch.
    expect(result.deleted).toBe(1);
    expect(result.rescheduled).toBe(true);

    await t.finishAllScheduledFunctions(vi.runAllTimers);
    const rows = await t.run((ctx) => ctx.db.query("intelHistory").collect());
    expect(rows).toHaveLength(0);
  });
});

describe("POST /relay/intel-history", () => {
  let originalRelay: string | undefined;

  beforeEach(() => {
    vi.useFakeTimers();
    vi.setSystemTime(NOW);
    originalRelay = process.env.RELAY_SHARED_SECRET;
    process.env.RELAY_SHARED_SECRET = RELAY_SECRET;
  });
  afterEach(() => {
    vi.useRealTimers();
    if (originalRelay === undefined) delete process.env.RELAY_SHARED_SECRET;
    else process.env.RELAY_SHARED_SECRET = originalRelay;
  });

  function post(body: unknown, secret: string | null = RELAY_SECRET) {
    const headers: Record<string, string> = { "Content-Type": "application/json" };
    if (secret !== null) headers.Authorization = `Bearer ${secret}`;
    return { method: "POST", headers, body: JSON.stringify(body) };
  }

  function ingestBody(records: AppendRecord[]) {
    return { domain: "conflict", resource: "conflict-events", runId: "run-1", records };
  }

  test.each([
    ["missing", null],
    ["wrong", "not-the-secret"],
  ])("rejects a %s bearer secret with 401", async (_label, secret) => {
    const t = convexTest(schema, modules);
    const res = await t.fetch(
      "/relay/intel-history",
      post(ingestBody([record()]), secret as string | null),
    );
    expect(res.status).toBe(401);
    expect(await res.json()).toEqual({ error: "UNAUTHORIZED" });
  });

  test("rejects a non-object body with 400", async () => {
    const t = convexTest(schema, modules);
    const res = await t.fetch("/relay/intel-history", post([1, 2, 3]));
    expect(res.status).toBe(400);
    expect(await res.json()).toEqual({ error: "INVALID_JSON" });
  });

  test("rejects a missing runId with 400", async () => {
    const t = convexTest(schema, modules);
    const res = await t.fetch(
      "/relay/intel-history",
      post({ domain: "conflict", resource: "conflict-events", records: [record()] }),
    );
    expect(res.status).toBe(400);
    expect((await res.json()).error).toBe("MISSING_FIELDS");
  });

  test("rejects more than 100 records with 400", async () => {
    const t = convexTest(schema, modules);
    const res = await t.fetch(
      "/relay/intel-history",
      post(
        ingestBody(
          Array.from({ length: INTEL_HISTORY_MAX_APPEND_RECORDS + 1 }, (_, i) =>
            record({ dedupeKey: `k-${i}` }),
          ),
        ),
      ),
    );
    expect(res.status).toBe(400);
    expect(await res.json()).toMatchObject({
      error: "TOO_MANY_RECORDS",
      max: INTEL_HISTORY_MAX_APPEND_RECORDS,
    });
  });

  test.each([
    ["short embedding", { embedding: [0.1] }],
    ["oversized title", { title: "t".repeat(501) }],
    ["oversized summary", { summary: "s".repeat(2001) }],
    ["missing dedupeKey", { dedupeKey: "" }],
    // The seeder already strips these, but this is the trust boundary: a
    // compromised relay credential must not be able to store a scheme the MCP
    // tools would publish to agents as a canonical link.
    ["javascript: sourceUrl", { sourceUrl: "javascript:alert(1)" }],
    ["data: sourceUrl", { sourceUrl: "data:text/html;base64,PHNjcmlwdD4=" }],
    ["protocol-relative sourceUrl", { sourceUrl: "//evil.test/x" }],
  ])("rejects a record with a %s (400 INVALID_RECORD)", async (_label, patch) => {
    const t = convexTest(schema, modules);
    const res = await t.fetch(
      "/relay/intel-history",
      post(ingestBody([record(patch as Partial<AppendRecord>)])),
    );
    expect(res.status).toBe(400);
    expect((await res.json()).error).toBe("INVALID_RECORD");

    const rows = await t.run((ctx) => ctx.db.query("intelHistory").collect());
    expect(rows).toHaveLength(0);
  });

  test("ingests records and reports the dedupe split on replay", async () => {
    const t = await intelHistoryAppendTest();
    const body = ingestBody([
      record({ dedupeKey: "h1", country: "UA" }),
      record({ dedupeKey: "h2", occurredAt: NOW - 2 * DAY }),
    ]);

    const first = await t.fetch("/relay/intel-history", post(body));
    expect(first.status).toBe(200);
    expect(await first.json()).toEqual({ inserted: 2, skipped: 0, retracted: 0 });

    const replay = await t.fetch("/relay/intel-history", post(body));
    expect(replay.status).toBe(200);
    expect(await replay.json()).toEqual({ inserted: 0, skipped: 2, retracted: 0 });

    const rows = await t.run((ctx) => ctx.db.query("intelHistory").collect());
    expect(rows).toHaveLength(2);
  });
});

describe("internal intel read routes", () => {
  let originalConvex: string | undefined;

  beforeEach(() => {
    vi.useFakeTimers();
    vi.setSystemTime(NOW);
    originalConvex = process.env.CONVEX_SERVER_SHARED_SECRET;
    process.env.CONVEX_SERVER_SHARED_SECRET = CONVEX_SECRET;
  });
  afterEach(() => {
    vi.useRealTimers();
    if (originalConvex === undefined) delete process.env.CONVEX_SERVER_SHARED_SECRET;
    else process.env.CONVEX_SERVER_SHARED_SECRET = originalConvex;
  });

  function post(body: unknown, secret: string | null = CONVEX_SECRET) {
    const headers: Record<string, string> = { "Content-Type": "application/json" };
    if (secret !== null) headers["x-convex-shared-secret"] = secret;
    return { method: "POST", headers, body: JSON.stringify(body) };
  }

  test.each([
    ["/api/internal-intel-timeline", { domain: "conflict" }],
    ["/api/internal-intel-search", { embedding: [] }],
  ])("%s rejects a missing shared secret with 401", async (path, body) => {
    const t = convexTest(schema, modules);
    const res = await t.fetch(path, post(body, null));
    expect(res.status).toBe(401);
    expect(await res.json()).toEqual({ error: "UNAUTHORIZED" });
  });

  test.each(["/api/internal-intel-timeline", "/api/internal-intel-search"])(
    "%s rejects a wrong shared secret with 401",
    async (path) => {
      const t = convexTest(schema, modules);
      const res = await t.fetch(path, post({}, "wrong-secret"));
      expect(res.status).toBe(401);
    },
  );

  test("timeline rejects an unscoped query with 400", async () => {
    const t = convexTest(schema, modules);
    const res = await t.fetch("/api/internal-intel-timeline", post({ from: NOW - DAY }));
    expect(res.status).toBe(400);
    expect((await res.json()).error).toBe("MISSING_SCOPE");
  });

  test("timeline returns projected records for a scoped query", async () => {
    const t = convexTest(schema, modules);
    await seed(t, [
      { dedupeKey: "t1", country: "UA", occurredAt: NOW - DAY, title: "recent" },
      { dedupeKey: "t2", country: "UA", occurredAt: NOW - 30 * DAY, title: "older" },
    ]);

    const res = await t.fetch(
      "/api/internal-intel-timeline",
      post({ domain: "conflict", country: "UA", from: NOW - 7 * DAY, to: NOW }),
    );

    expect(res.status).toBe(200);
    const body = (await res.json()) as { records: Array<Record<string, unknown>> };
    expect(body.records.map((r) => r.title)).toEqual(["recent"]);
    expect(body.records[0]).not.toHaveProperty("embedding");
  });

  test("search rejects a wrong-dimension embedding with 400", async () => {
    const t = convexTest(schema, modules);
    const res = await t.fetch(
      "/api/internal-intel-search",
      post({ embedding: [0.1, 0.2] }),
    );
    expect(res.status).toBe(400);
    expect((await res.json()).error).toBe("INVALID_EMBEDDING");
  });

  test("search returns scored records", async () => {
    const t = convexTest(schema, modules);
    await seed(t, [
      { dedupeKey: "v1", title: "exact", occurredAt: NOW - DAY, embedding: unitVector(0) },
      { dedupeKey: "v2", title: "far", occurredAt: NOW - DAY, embedding: unitVector(9) },
    ]);

    const res = await t.fetch(
      "/api/internal-intel-search",
      post({ embedding: unitVector(0), domain: "conflict", limit: 5 }),
    );

    expect(res.status).toBe(200);
    const body = (await res.json()) as {
      records: Array<{ title: string; _score: number }>;
    };
    expect(body.records.map((r) => r.title)).toEqual(["exact", "far"]);
    expect(body.records[0]._score).toBeCloseTo(1, 6);
  });

  test("search forwards a valid score floor and rejects invalid values", async () => {
    const t = convexTest(schema, modules);
    await seed(t, [
      { dedupeKey: "v1", title: "exact", occurredAt: NOW - DAY, embedding: unitVector(0) },
      { dedupeKey: "v2", title: "far", occurredAt: NOW - DAY, embedding: unitVector(9) },
    ]);

    const filtered = await t.fetch(
      "/api/internal-intel-search",
      post({ embedding: unitVector(0), domain: "conflict", limit: 5, minScore: 0.55 }),
    );
    expect(filtered.status).toBe(200);
    expect(
      ((await filtered.json()) as { records: Array<{ title: string }> }).records.map(
        (record) => record.title,
      ),
    ).toEqual(["exact"]);

    for (const minScore of [-1.1, 1.1, Number.NaN, "0.55"]) {
      const rejected = await t.fetch(
        "/api/internal-intel-search",
        post({ embedding: unitVector(0), domain: "conflict", minScore }),
      );
      expect(rejected.status).toBe(400);
      expect((await rejected.json()).error).toBe("INVALID_MIN_SCORE");
    }
  });
});

// ---------------------------------------------------------------------------
// Retraction (#5743)
//
// The store is agent-facing and durable for 180 days, so a poisoned or wrong
// feed item outlives by months the live snapshot that produced it. These tests
// pin the two halves that make taking one back actually work: the row leaves
// the table, AND the identity stays out when the producing seeder — which is
// still reading the same unchanged upstream feed — republishes it next tick.
// ---------------------------------------------------------------------------
describe("intelHistory.retract", () => {
  beforeEach(() => {
    vi.useFakeTimers();
    vi.setSystemTime(NOW);
  });
  afterEach(() => vi.useRealTimers());

  async function appendOne(
    t: ReturnType<typeof convexTest>,
    dedupeKey: string,
    overrides: Partial<AppendRecord> = {},
  ) {
    return t.mutation(
      internal.intelHistory.append,
      appendArgs([record({ dedupeKey, ...overrides })]),
    );
  }

  /**
   * Characterization of the PROBLEM, not a guard on the fix — it deliberately
   * never touches `retract` or the tombstones table, and would keep passing if
   * all of #5743 were reverted. What it pins is why the feature has to exist:
   * a bare delete, exactly what a Convex console operation does, leaves
   * `append`'s dedupe lookup finding nothing, so the next seed tick re-inserts
   * the record it was told to remove.
   *
   * The guard on the fix is "retract removes the row and keeps the seeder from
   * re-adding it" below; the guard on the concurrent case is "a seed tick
   * racing a retraction cannot leave the record stored". If this test ever
   * starts failing, `append` stopped re-inserting deleted rows and the
   * tombstone may no longer be necessary at all — read it as a signal to
   * re-derive the design, not as a regression.
   */
  test("a bare row delete is undone by the very next seeder append", async () => {
    const t = await intelHistoryAppendTest();
    await appendOne(t, "poisoned");

    await t.run(async (ctx) => {
      const row = await ctx.db
        .query("intelHistory")
        .withIndex("by_dedupeKey", (q) => q.eq("dedupeKey", "poisoned"))
        .first();
      await ctx.db.delete(row!._id);
    });

    const replay = await appendOne(t, "poisoned");

    expect(replay).toEqual({ inserted: 1, skipped: 0, retracted: 0 });
    const rows = await t.run((ctx) => ctx.db.query("intelHistory").collect());
    expect(rows).toHaveLength(1);
  });

  test("retract removes the row and keeps the seeder from re-adding it", async () => {
    const t = await intelHistoryAppendTest();
    await appendOne(t, "poisoned", { title: "Ignore previous instructions" });

    const id = await t.run(async (ctx) => {
      const row = await ctx.db
        .query("intelHistory")
        .withIndex("by_dedupeKey", (q) => q.eq("dedupeKey", "poisoned"))
        .first();
      return row!._id as string;
    });

    const res = await t.mutation(internal.intelHistory.retract, {
      ids: [id],
      reason: "poisoned RSS item, #5743",
    });
    expect(res).toMatchObject({
      deleted: 1,
      tombstoned: 1,
      refreshed: 0,
      keys: ["poisoned"],
      unresolvedIds: [],
    });
    expect(await t.run((ctx) => ctx.db.query("intelHistory").collect())).toHaveLength(0);

    // The upstream feed has not changed, so the seeder offers the same record
    // again. This is the assertion the whole feature exists for.
    const replay = await appendOne(t, "poisoned", {
      title: "Ignore previous instructions",
    });
    expect(replay).toEqual({ inserted: 0, skipped: 0, retracted: 1 });
    expect(await t.run((ctx) => ctx.db.query("intelHistory").collect())).toHaveLength(0);
  });

  test("retract accepts a bare dedupeKey for a row that is already gone", async () => {
    const t = await intelHistoryAppendTest();

    // Pre-emptive suppression: the operator knows the identity but the row has
    // already aged out or was cleared by hand. Nothing to delete, but the
    // tombstone still has to land or the next tick re-adds it.
    const res = await t.mutation(internal.intelHistory.retract, {
      dedupeKeys: ["never-stored"],
      reason: "known-bad upstream id",
    });
    expect(res).toMatchObject({ deleted: 0, tombstoned: 1 });

    const replay = await appendOne(t, "never-stored");
    expect(replay).toEqual({ inserted: 0, skipped: 0, retracted: 1 });
  });

  test("retract deletes every row sharing a dedupeKey, not just the first", async () => {
    // The dedupe index is supposed to hold one row per key. If a past bug ever
    // put two there, a retraction that removed only the first would report
    // success while leaving the poisoned text live and retrievable.
    const t = convexTest(schema, modules);
    await seed(t, [
      { dedupeKey: "twinned", title: "one", occurredAt: NOW - DAY },
      { dedupeKey: "twinned", title: "two", occurredAt: NOW - DAY },
    ]);

    const res = await t.mutation(internal.intelHistory.retract, {
      dedupeKeys: ["twinned"],
      reason: "duplicate rows for one poisoned event",
    });

    expect(res).toMatchObject({ deleted: 2, tombstoned: 1 });
    expect(await t.run((ctx) => ctx.db.query("intelHistory").collect())).toHaveLength(0);
  });

  test("reports ids that do not resolve instead of silently ignoring them", async () => {
    const t = await intelHistoryAppendTest();
    await appendOne(t, "live");
    const liveId = await t.run(async (ctx) => {
      const row = await ctx.db
        .query("intelHistory")
        .withIndex("by_dedupeKey", (q) => q.eq("dedupeKey", "live"))
        .first();
      return row!._id as string;
    });

    const res = await t.mutation(internal.intelHistory.retract, {
      ids: [liveId, "not-a-convex-id"],
      reason: "mixed batch",
    });

    expect(res.deleted).toBe(1);
    expect(res.unresolvedIds).toEqual(["not-a-convex-id"]);

    // Confirm the DB actually reached the reported state rather than trusting
    // the return: the live id's row is gone and its identity is tombstoned,
    // while the unresolved one contributed no tombstone at all.
    expect(await t.run((ctx) => ctx.db.query("intelHistory").collect())).toHaveLength(0);
    const tombstones = await t.run((ctx) =>
      ctx.db.query("intelHistoryRetractions").collect(),
    );
    expect(tombstones.map((row) => row.dedupeKey)).toEqual(["live"]);
  });

  test("re-retracting an existing tombstone refreshes it rather than duplicating", async () => {
    const t = await intelHistoryAppendTest();

    await t.mutation(internal.intelHistory.retract, {
      dedupeKeys: ["repeat"],
      reason: "first call",
    });
    vi.setSystemTime(NOW + 5 * DAY);
    const second = await t.mutation(internal.intelHistory.retract, {
      dedupeKeys: ["repeat"],
      reason: "still live upstream",
    });

    expect(second).toMatchObject({ tombstoned: 0, refreshed: 1 });
    const tombstones = await t.run((ctx) =>
      ctx.db.query("intelHistoryRetractions").collect(),
    );
    expect(tombstones).toHaveLength(1);
    // The clock restarts: an operator repeating the call is saying the item is
    // still being served, which is exactly when expiry would be premature.
    expect(tombstones[0]).toMatchObject({
      retractedAt: NOW + 5 * DAY,
      reason: "still live upstream",
    });
  });

  test("rejects an empty or oversized identifier batch", async () => {
    const t = await intelHistoryAppendTest();

    await expect(
      t.mutation(internal.intelHistory.retract, { reason: "nothing named" }),
    ).rejects.toThrow(/at least one of ids or dedupeKeys/i);

    await expect(
      t.mutation(internal.intelHistory.retract, {
        dedupeKeys: Array.from(
          { length: INTEL_HISTORY_MAX_RETRACT_IDENTIFIERS + 1 },
          (_, i) => `k-${i}`,
        ),
        reason: "too many",
      }),
    ).rejects.toThrow(/at most 100 identifiers/i);

    expect(
      await t.run((ctx) => ctx.db.query("intelHistoryRetractions").collect()),
    ).toHaveLength(0);
  });

  test("rejects a blank reason", async () => {
    const t = await intelHistoryAppendTest();
    await expect(
      t.mutation(internal.intelHistory.retract, {
        dedupeKeys: ["x"],
        reason: "   ",
      }),
    ).rejects.toThrow(/reason is required/i);
  });

  test("works without the deploy-seeded append lock", async () => {
    // append throws without the lock, so there is no concurrent writer to
    // serialize against — refusing to retract in that state would block an
    // incident cleanup for a race that cannot happen.
    const t = convexTest(schema, modules);
    await seed(t, [{ dedupeKey: "orphan", title: "orphan", occurredAt: NOW - DAY }]);

    const res = await t.mutation(internal.intelHistory.retract, {
      dedupeKeys: ["orphan"],
      reason: "lock absent",
    });

    expect(res).toMatchObject({ deleted: 1, tombstoned: 1 });
  });

  test("a retracted key blocks only itself inside a mixed batch", async () => {
    const t = await intelHistoryAppendTest();
    await t.mutation(internal.intelHistory.retract, {
      dedupeKeys: ["bad"],
      reason: "poisoned",
    });

    const res = await t.mutation(
      internal.intelHistory.append,
      appendArgs([
        record({ dedupeKey: "bad" }),
        record({ dedupeKey: "good-1" }),
        record({ dedupeKey: "good-2" }),
      ]),
    );

    expect(res).toEqual({ inserted: 2, skipped: 0, retracted: 1 });
    const rows = await t.run((ctx) => ctx.db.query("intelHistory").collect());
    expect(rows.map((r) => r.dedupeKey).sort()).toEqual(["good-1", "good-2"]);
  });
});

describe("intelHistory.restore", () => {
  beforeEach(() => {
    vi.useFakeTimers();
    vi.setSystemTime(NOW);
  });
  afterEach(() => vi.useRealTimers());

  test("lifts the tombstone so the seeder may store the event again", async () => {
    const t = await intelHistoryAppendTest();
    await t.mutation(internal.intelHistory.retract, {
      dedupeKeys: ["mistake"],
      reason: "retracted in error",
    });

    const res = await t.mutation(internal.intelHistory.restore, {
      dedupeKeys: ["mistake"],
    });
    expect(res).toEqual({ removed: 1, notRetracted: [] });

    // Restore does NOT resurrect the deleted row — its embedding is gone. What
    // it restores is the seeder's ability to append the event next tick.
    const replay = await t.mutation(
      internal.intelHistory.append,
      appendArgs([record({ dedupeKey: "mistake" })]),
    );
    expect(replay).toEqual({ inserted: 1, skipped: 0, retracted: 0 });
  });

  test("removes every duplicate tombstone before allowing append", async () => {
    const t = await intelHistoryAppendTest();
    await t.run(async (ctx) => {
      await ctx.db.insert("intelHistoryRetractions", {
        dedupeKey: "duplicate-tombstone",
        retractedAt: NOW - DAY,
        reason: "first",
      });
      await ctx.db.insert("intelHistoryRetractions", {
        dedupeKey: "duplicate-tombstone",
        retractedAt: NOW,
        reason: "duplicate",
      });
    });

    const res = await t.mutation(internal.intelHistory.restore, {
      dedupeKeys: ["duplicate-tombstone"],
    });
    expect(res).toEqual({ removed: 2, notRetracted: [] });

    const tombstones = await t.run((ctx) =>
      ctx.db
        .query("intelHistoryRetractions")
        .withIndex("by_dedupeKey", (q) => q.eq("dedupeKey", "duplicate-tombstone"))
        .collect(),
    );
    expect(tombstones).toHaveLength(0);

    const replay = await t.mutation(
      internal.intelHistory.append,
      appendArgs([record({ dedupeKey: "duplicate-tombstone" })]),
    );
    expect(replay).toEqual({ inserted: 1, skipped: 0, retracted: 0 });
  });

  test("reports keys that were never retracted", async () => {
    const t = await intelHistoryAppendTest();
    const res = await t.mutation(internal.intelHistory.restore, {
      dedupeKeys: ["unknown"],
    });
    expect(res).toEqual({ removed: 0, notRetracted: ["unknown"] });
  });
});

describe("intelHistory.listRetractions", () => {
  beforeEach(() => {
    vi.useFakeTimers();
    vi.setSystemTime(NOW);
  });
  afterEach(() => vi.useRealTimers());

  test("returns tombstones newest first, bounded by the page cap", async () => {
    const t = await intelHistoryAppendTest();
    for (const [index, key] of ["oldest", "middle", "newest"].entries()) {
      vi.setSystemTime(NOW + index * DAY);
      await t.mutation(internal.intelHistory.retract, {
        dedupeKeys: [key],
        reason: `reason-${key}`,
      });
    }

    const all = await t.query(internal.intelHistory.listRetractions, {});
    expect(all.retractions.map((r) => r.dedupeKey)).toEqual([
      "newest",
      "middle",
      "oldest",
    ]);
    expect(all.retractions[0]).toMatchObject({ reason: "reason-newest" });

    const paged = await t.query(internal.intelHistory.listRetractions, { limit: 1 });
    expect(paged.retractions).toHaveLength(1);
  });
});

describe("intelHistory.prune — retraction tombstones", () => {
  beforeEach(() => vi.useFakeTimers());
  afterEach(() => {
    vi.clearAllTimers();
    vi.useRealTimers();
  });

  test("ages tombstones out on the retention clock, measured from retractedAt", async () => {
    const t = convexTest(schema, modules);
    const beyond = NOW - (INTEL_HISTORY_RETENTION_DAYS + 1) * DAY;
    await t.run(async (ctx) => {
      await ctx.db.insert("intelHistoryRetractions", {
        dedupeKey: "long-expired",
        retractedAt: beyond,
        reason: "old incident",
      });
      await ctx.db.insert("intelHistoryRetractions", {
        dedupeKey: "still-suppressed",
        retractedAt: NOW - DAY,
        reason: "recent incident",
      });
    });

    const res = await t.mutation(internal.intelHistory.prune, { now: NOW });

    expect(res).toEqual({ deleted: 0, deletedRetractions: 1, rescheduled: false });
    const left = await t.run((ctx) =>
      ctx.db.query("intelHistoryRetractions").collect(),
    );
    expect(left.map((r) => r.dedupeKey)).toEqual(["still-suppressed"]);
  });

  test("self-drains a tombstone backlog even when no history rows are stale", async () => {
    const t = convexTest(schema, modules);
    const beyond = NOW - (INTEL_HISTORY_RETENTION_DAYS + 1) * DAY;
    await t.run(async (ctx) => {
      for (let i = 0; i < 3; i++) {
        await ctx.db.insert("intelHistoryRetractions", {
          dedupeKey: `expired-${i}`,
          retractedAt: beyond,
          reason: "old",
        });
      }
    });

    const first = await t.mutation(internal.intelHistory.prune, {
      now: NOW,
      limit: 2,
    });
    expect(first).toEqual({ deleted: 0, deletedRetractions: 2, rescheduled: true });

    await t.finishAllScheduledFunctions(vi.runAllTimers);
    expect(
      await t.run((ctx) => ctx.db.query("intelHistoryRetractions").collect()),
    ).toHaveLength(0);
  });
});

describe("POST /relay/intel-history/retract", () => {
  let originalRelay: string | undefined;
  let originalRetract: string | undefined;

  beforeEach(() => {
    vi.useFakeTimers();
    vi.setSystemTime(NOW);
    originalRelay = process.env.RELAY_SHARED_SECRET;
    originalRetract = process.env.RELAY_RETRACT_SECRET;
    process.env.RELAY_SHARED_SECRET = RELAY_SECRET;
    process.env.RELAY_RETRACT_SECRET = RELAY_SECRET;
  });
  afterEach(() => {
    vi.useRealTimers();
    if (originalRelay === undefined) delete process.env.RELAY_SHARED_SECRET;
    else process.env.RELAY_SHARED_SECRET = originalRelay;
    if (originalRetract === undefined) delete process.env.RELAY_RETRACT_SECRET;
    else process.env.RELAY_RETRACT_SECRET = originalRetract;
  });

  function post(body: unknown, secret: string | null = RELAY_SECRET) {
    const headers: Record<string, string> = { "Content-Type": "application/json" };
    if (secret !== null) headers.Authorization = `Bearer ${secret}`;
    return { method: "POST", headers, body: JSON.stringify(body) };
  }

  test.each([
    ["/relay/intel-history/retract", { dedupeKeys: ["k"], reason: "r" }],
    ["/relay/intel-history/restore", { dedupeKeys: ["k"] }],
    ["/relay/intel-history/retractions", {}],
  ])("%s rejects a missing bearer secret with 401", async (path, body) => {
    const t = convexTest(schema, modules);
    const res = await t.fetch(path, post(body, null));
    expect(res.status).toBe(401);
    expect(await res.json()).toEqual({ error: "UNAUTHORIZED" });
  });

  test.each([
    ["/relay/intel-history/retract", { dedupeKeys: ["k"], reason: "r" }],
    ["/relay/intel-history/restore", { dedupeKeys: ["k"] }],
  ])("%s rejects a wrong bearer secret with 401", async (path, body) => {
    const t = convexTest(schema, modules);
    const res = await t.fetch(path, post(body, "not-the-secret"));
    expect(res.status).toBe(401);

    expect(
      await t.run((ctx) => ctx.db.query("intelHistoryRetractions").collect()),
    ).toHaveLength(0);
  });

  test("retracts through the route and reports the effect", async () => {
    const t = await intelHistoryAppendTest();
    await t.mutation(
      internal.intelHistory.append,
      appendArgs([record({ dedupeKey: "wire-poisoned" })]),
    );

    const res = await t.fetch(
      "/relay/intel-history/retract",
      post({ dedupeKeys: ["wire-poisoned"], reason: "poisoned feed item #5743" }),
    );

    expect(res.status).toBe(200);
    expect(await res.json()).toMatchObject({ deleted: 1, tombstoned: 1 });
    expect(await t.run((ctx) => ctx.db.query("intelHistory").collect())).toHaveLength(0);
  });

  test.each([
    ["no identifiers", { reason: "r" }, "MISSING_IDENTIFIERS"],
    ["blank reason", { dedupeKeys: ["k"], reason: "  " }, "MISSING_REASON"],
    ["absent reason", { dedupeKeys: ["k"] }, "MISSING_REASON"],
    ["non-array ids", { ids: "abc", reason: "r" }, "INVALID_IDS"],
    ["empty-string key", { dedupeKeys: [""], reason: "r" }, "INVALID_DEDUPE_KEYS"],
    ["non-string key", { dedupeKeys: [7], reason: "r" }, "INVALID_DEDUPE_KEYS"],
  ])("rejects %s with 400", async (_label, body, error) => {
    const t = convexTest(schema, modules);
    const res = await t.fetch("/relay/intel-history/retract", post(body));
    expect(res.status).toBe(400);
    expect((await res.json()).error).toBe(error);
  });

  test("rejects more identifiers than the per-call cap", async () => {
    const t = convexTest(schema, modules);
    const res = await t.fetch(
      "/relay/intel-history/retract",
      post({
        dedupeKeys: Array.from(
          { length: INTEL_HISTORY_MAX_RETRACT_IDENTIFIERS + 1 },
          (_, i) => `k-${i}`,
        ),
        reason: "sweep",
      }),
    );
    expect(res.status).toBe(400);
    expect(await res.json()).toMatchObject({
      error: "TOO_MANY_IDENTIFIERS",
      max: INTEL_HISTORY_MAX_RETRACT_IDENTIFIERS,
    });
  });

  test("restore lifts a tombstone and lists what remains", async () => {
    const t = await intelHistoryAppendTest();
    await t.fetch(
      "/relay/intel-history/retract",
      post({ dedupeKeys: ["a", "b"], reason: "batch" }),
    );

    const restored = await t.fetch(
      "/relay/intel-history/restore",
      post({ dedupeKeys: ["a"] }),
    );
    expect(restored.status).toBe(200);
    expect(await restored.json()).toEqual({ removed: 1, notRetracted: [] });

    const listed = await t.fetch("/relay/intel-history/retractions", post({}));
    expect(listed.status).toBe(200);
    const body = (await listed.json()) as {
      retractions: Array<{ dedupeKey: string }>;
    };
    expect(body.retractions.map((r) => r.dedupeKey)).toEqual(["b"]);
  });
});

// ---------------------------------------------------------------------------
// Post-review hardening (#5743)
//
// Each block below pins a property an earlier revision got wrong or left
// unasserted. The concurrency case is the important one: `touchAppendLock` in
// `retract` is the single line stopping a seed tick from re-inserting a record
// mid-retraction, and before this test the suite stayed green with it deleted.
// ---------------------------------------------------------------------------
describe("intelHistory.retract — concurrency and identifier resolution", () => {
  beforeEach(() => {
    vi.useFakeTimers();
    vi.setSystemTime(NOW);
  });
  afterEach(() => vi.useRealTimers());

  test("a seed tick racing a retraction cannot leave the record stored", async () => {
    const t = await intelHistoryAppendTest();

    // Whichever transaction wins, the OCC dependency on the shared append lock
    // forces the loser to retry against the winner's writes: append-then-
    // retract ends with the row deleted, retract-then-append ends with the
    // append skipped by the tombstone. Both land on the same invariant, which
    // is what makes this assertable without controlling the interleaving.
    await Promise.all([
      t.mutation(
        internal.intelHistory.append,
        appendArgs([record({ dedupeKey: "racing" })]),
      ),
      t.mutation(internal.intelHistory.retract, {
        dedupeKeys: ["racing"],
        reason: "retracted mid-run",
      }),
    ]);

    const rows = await t.run((ctx) =>
      ctx.db
        .query("intelHistory")
        .withIndex("by_dedupeKey", (q) => q.eq("dedupeKey", "racing"))
        .collect(),
    );
    expect(rows).toHaveLength(0);
    const tombstones = await t.run((ctx) =>
      ctx.db.query("intelHistoryRetractions").collect(),
    );
    expect(tombstones).toHaveLength(1);
  });

  test("retract touches the append lock so the OCC dependency exists at all", async () => {
    const t = await intelHistoryAppendTest();
    const before = await t.run(async (ctx) => {
      const lock = await ctx.db.query("intelHistoryAppendLocks").first();
      await ctx.db.patch(lock!._id, { lastTouchedAt: NOW - 10 * DAY });
      return NOW - 10 * DAY;
    });

    await t.mutation(internal.intelHistory.retract, {
      dedupeKeys: ["k"],
      reason: "r",
    });

    const after = await t.run(async (ctx) => {
      const lock = await ctx.db.query("intelHistoryAppendLocks").first();
      return lock!.lastTouchedAt;
    });
    expect(after).toBeGreaterThan(before);
  });

  test("rejects a batch whose ids all fail to resolve instead of reporting success", async () => {
    // The operator's most likely workflow is copying `id` out of a search
    // result. If the row was pruned or already retracted since, nothing
    // resolves, nothing is tombstoned, and a 200 would read as "already
    // clean" while the next seed tick re-adds the record.
    const t = await intelHistoryAppendTest();

    await expect(
      t.mutation(internal.intelHistory.retract, {
        ids: ["not-a-convex-id"],
        reason: "stale id",
      }),
    ).rejects.toThrow(/no identifier resolved to a dedupeKey/i);

    expect(
      await t.run((ctx) => ctx.db.query("intelHistoryRetractions").collect()),
    ).toHaveLength(0);
  });

  test("resolves a mixed ids + dedupeKeys batch into one deduplicated key set", async () => {
    const t = await intelHistoryAppendTest();
    await t.mutation(
      internal.intelHistory.append,
      appendArgs([record({ dedupeKey: "shared" })]),
    );
    const id = await t.run(async (ctx) => {
      const row = await ctx.db
        .query("intelHistory")
        .withIndex("by_dedupeKey", (q) => q.eq("dedupeKey", "shared"))
        .first();
      return row!._id as string;
    });

    // The id and the dedupeKey name the SAME record; the Set in
    // resolveRetractionKeys must collapse them rather than tombstone twice.
    const res = await t.mutation(internal.intelHistory.retract, {
      ids: [id],
      dedupeKeys: ["shared", "other"],
      reason: "mixed batch",
    });

    expect(res.keys.sort()).toEqual(["other", "shared"]);
    expect(res).toMatchObject({ deleted: 1, tombstoned: 2, refreshed: 0 });
    expect(
      await t.run((ctx) => ctx.db.query("intelHistoryRetractions").collect()),
    ).toHaveLength(2);
  });
});

describe("intelHistory tombstone lifetime", () => {
  beforeEach(() => {
    vi.useFakeTimers();
    vi.setSystemTime(NOW);
  });
  afterEach(() => {
    vi.clearAllTimers();
    vi.useRealTimers();
  });

  test("a suppressed append refreshes the tombstone, so expiry tracks the producer", async () => {
    const t = await intelHistoryAppendTest();
    await t.mutation(internal.intelHistory.retract, {
      dedupeKeys: ["still-upstream"],
      reason: "poisoned",
    });

    // 179 days later the feed is STILL serving the item. That attempt is the
    // evidence the tombstone must outlive, so it restarts the clock — the
    // alternative is a record that silently returns on day 181.
    vi.setSystemTime(NOW + 179 * DAY);
    const suppressed = await t.mutation(
      internal.intelHistory.append,
      appendArgs([record({ dedupeKey: "still-upstream" })]),
    );
    expect(suppressed).toEqual({ inserted: 0, skipped: 0, retracted: 1 });

    const tombstone = await t.run((ctx) =>
      ctx.db.query("intelHistoryRetractions").first(),
    );
    expect(tombstone!.retractedAt).toBe(NOW + 179 * DAY);

    // Day 181 from the ORIGINAL retraction would have pruned it; from the
    // refreshed one it survives, and the record stays suppressed.
    const pruned = await t.mutation(internal.intelHistory.prune, {
      now: NOW + 181 * DAY,
    });
    expect(pruned.deletedRetractions).toBe(0);
  });

  test("a shortened retentionMs cannot drain tombstones early", async () => {
    // `retentionMs` is an operator-callable override copied from the
    // apiPlanLimit prune, where shortening it just deletes old rows early.
    // Here it is a security control: prune({retentionMs: 0}) would set the
    // cutoff to `now`, drain every tombstone, and hand every retracted
    // identity back to its producing feed on the next tick.
    const t = await intelHistoryAppendTest();
    await t.mutation(internal.intelHistory.retract, {
      dedupeKeys: ["suppressed"],
      reason: "poisoned",
    });

    // Prune strictly AFTER the tombstone was written. Pruning at exactly NOW
    // would pass even with the clamp removed — the cutoff is a strict `<`, so
    // a tombstone stamped NOW is not older than a cutoff of NOW, and the test
    // would assert nothing.
    for (const retentionMs of [0, 1, 60_000]) {
      const res = await t.mutation(internal.intelHistory.prune, {
        now: NOW + DAY,
        retentionMs,
      });
      expect(res.deletedRetractions, `retentionMs ${retentionMs} must not drain`).toBe(0);
    }

    expect(
      await t.run((ctx) => ctx.db.query("intelHistoryRetractions").collect()),
    ).toHaveLength(1);
    // ...and the suppression still holds.
    const replay = await t.mutation(
      internal.intelHistory.append,
      appendArgs([record({ dedupeKey: "suppressed" })]),
    );
    expect(replay).toEqual({ inserted: 0, skipped: 0, retracted: 1 });
  });

  test("a tombstone the producer stopped hitting expires and the record may return", async () => {
    // The honest other half: suppression is not permanent. Once the feed stops
    // offering the item, nothing refreshes the tombstone, prune drains it, and
    // a later re-publish is stored again. Pinned so the 180-day bet is a
    // documented behaviour rather than a surprise.
    const t = await intelHistoryAppendTest();
    await t.mutation(internal.intelHistory.retract, {
      dedupeKeys: ["gone-upstream"],
      reason: "poisoned",
    });

    const pruned = await t.mutation(internal.intelHistory.prune, {
      now: NOW + (INTEL_HISTORY_RETENTION_DAYS + 1) * DAY,
    });
    expect(pruned.deletedRetractions).toBe(1);

    const replay = await t.mutation(
      internal.intelHistory.append,
      appendArgs([record({ dedupeKey: "gone-upstream" })]),
    );
    expect(replay).toEqual({ inserted: 1, skipped: 0, retracted: 0 });
  });

  test("prune drains both tables in one pass when each is at its batch cap", async () => {
    const t = convexTest(schema, modules);
    const beyond = NOW - (INTEL_HISTORY_RETENTION_DAYS + 1) * DAY;
    await seed(
      t,
      Array.from({ length: 2 }, (_, i) => ({
        dedupeKey: `aged-${i}`,
        ingestedAt: beyond,
        occurredAt: beyond,
        title: `aged-${i}`,
      })),
    );
    await t.run(async (ctx) => {
      for (let i = 0; i < 2; i++) {
        await ctx.db.insert("intelHistoryRetractions", {
          dedupeKey: `expired-${i}`,
          retractedAt: beyond,
          reason: "old",
        });
      }
    });

    const first = await t.mutation(internal.intelHistory.prune, {
      now: NOW,
      limit: 2,
    });
    expect(first).toEqual({ deleted: 2, deletedRetractions: 2, rescheduled: true });

    await t.finishAllScheduledFunctions(vi.runAllTimers);
    expect(await t.run((ctx) => ctx.db.query("intelHistory").collect())).toHaveLength(0);
    expect(
      await t.run((ctx) => ctx.db.query("intelHistoryRetractions").collect()),
    ).toHaveLength(0);
  });
});

describe("intelHistory.restore — argument handling", () => {
  beforeEach(() => {
    vi.useFakeTimers();
    vi.setSystemTime(NOW);
  });
  afterEach(() => vi.useRealTimers());

  test("a repeated key is not reported as both removed and never-retracted", async () => {
    const t = await intelHistoryAppendTest();
    await t.mutation(internal.intelHistory.retract, {
      dedupeKeys: ["dup"],
      reason: "r",
    });

    const res = await t.mutation(internal.intelHistory.restore, {
      dedupeKeys: ["dup", "dup"],
    });

    expect(res).toEqual({ removed: 1, notRetracted: [] });
  });

  test("names only the arguments it actually accepts when given none", async () => {
    // restore takes no ids, so pointing its caller at one sends them after an
    // argument the mutation would reject.
    const t = await intelHistoryAppendTest();
    await expect(
      t.mutation(internal.intelHistory.restore, { dedupeKeys: [] }),
    ).rejects.toThrow(/at least one of dedupeKeys is required/i);
  });

  test("enforces the same per-call identifier cap as retract", async () => {
    const t = await intelHistoryAppendTest();
    await expect(
      t.mutation(internal.intelHistory.restore, {
        dedupeKeys: Array.from(
          { length: INTEL_HISTORY_MAX_RETRACT_IDENTIFIERS + 1 },
          (_, i) => `k-${i}`,
        ),
      }),
    ).rejects.toThrow(/at most 100 identifiers/i);
  });
});

describe("intelHistory.listRetractions — truncation signal", () => {
  beforeEach(() => {
    vi.useFakeTimers();
    vi.setSystemTime(NOW);
  });
  afterEach(() => vi.useRealTimers());

  test("marks a filled page partial so a truncated review is not read as exhaustive", async () => {
    const t = await intelHistoryAppendTest();
    await t.mutation(internal.intelHistory.retract, {
      dedupeKeys: ["a", "b", "c"],
      reason: "batch",
    });

    const full = await t.query(internal.intelHistory.listRetractions, {});
    expect(full.partial).toBe(false);

    const page = await t.query(internal.intelHistory.listRetractions, { limit: 2 });
    expect(page.retractions).toHaveLength(2);
    expect(page.partial).toBe(true);
  });
});

describe("relay retraction routes — validation edges", () => {
  let originalRelay: string | undefined;
  let originalRetract: string | undefined;

  beforeEach(() => {
    vi.useFakeTimers();
    vi.setSystemTime(NOW);
    originalRelay = process.env.RELAY_SHARED_SECRET;
    originalRetract = process.env.RELAY_RETRACT_SECRET;
    process.env.RELAY_SHARED_SECRET = RELAY_SECRET;
    process.env.RELAY_RETRACT_SECRET = RELAY_SECRET;
  });
  afterEach(() => {
    vi.useRealTimers();
    if (originalRelay === undefined) delete process.env.RELAY_SHARED_SECRET;
    else process.env.RELAY_SHARED_SECRET = originalRelay;
    if (originalRetract === undefined) delete process.env.RELAY_RETRACT_SECRET;
    else process.env.RELAY_RETRACT_SECRET = originalRetract;
  });

  function post(body: unknown, secret: string | null = RELAY_SECRET) {
    const headers: Record<string, string> = { "Content-Type": "application/json" };
    if (secret !== null) headers.Authorization = `Bearer ${secret}`;
    return { method: "POST", headers, body: JSON.stringify(body) };
  }

  test("rejects an untrimmed identifier instead of tombstoning the typo", async () => {
    // A stray space from a copy-paste makes retract look up an identity that
    // does not exist: nothing is deleted, a tombstone is written for the typo,
    // and the caller reads "tombstoned: 1" while the real record stays live.
    const t = await intelHistoryAppendTest();

    for (const bad of [" energy:intelligence:abc", "energy:intelligence:abc ", " "]) {
      const res = await t.fetch(
        "/relay/intel-history/retract",
        post({ dedupeKeys: [bad], reason: "r" }),
      );
      expect(res.status, `${JSON.stringify(bad)} must be rejected`).toBe(400);
      expect((await res.json()).error).toBe("INVALID_DEDUPE_KEYS");
    }

    const withId = await t.fetch(
      "/relay/intel-history/retract",
      post({ ids: [" abc "], reason: "r" }),
    );
    expect(withId.status).toBe(400);
    expect((await withId.json()).error).toBe("INVALID_IDS");

    // Nothing was written on any of those attempts.
    expect(
      await t.run((ctx) => ctx.db.query("intelHistoryRetractions").collect()),
    ).toHaveLength(0);
  });

  test("rejects an over-length identifier rather than silently dropping it", async () => {
    // A dropped identifier would make the route report success for a
    // retraction that left the named record live.
    const t = convexTest(schema, modules);

    const longKey = await t.fetch(
      "/relay/intel-history/retract",
      post({ dedupeKeys: ["k".repeat(257)], reason: "r" }),
    );
    expect(longKey.status).toBe(400);
    expect((await longKey.json()).error).toBe("INVALID_DEDUPE_KEYS");

    const longId = await t.fetch(
      "/relay/intel-history/retract",
      post({ ids: ["i".repeat(129)], reason: "r" }),
    );
    expect(longId.status).toBe(400);
    expect((await longId.json()).error).toBe("INVALID_IDS");
  });

  test("rejects a malformed limit on the retractions review route", async () => {
    const t = convexTest(schema, modules);
    const res = await t.fetch(
      "/relay/intel-history/retractions",
      post({ limit: "twenty" }),
    );
    expect(res.status).toBe(400);
    expect((await res.json()).error).toBe("INVALID_LIMIT");
  });

  test("surfaces an all-unresolved retraction as an error, not a 200", async () => {
    const t = await intelHistoryAppendTest();
    const res = await t.fetch(
      "/relay/intel-history/retract",
      post({ ids: ["not-a-convex-id"], reason: "stale id" }),
    );
    expect(res.status).toBe(500);
    expect((await res.json()).error).toMatch(/no identifier resolved/i);
  });

  test("RELAY_RETRACT_SECRET keeps the fleet secret from gaining delete power", async () => {
    // RELAY_SHARED_SECRET is held by every Railway seeder that appends
    // history. Separating the credentials is only worth anything if setting
    // the retraction secret actually CLOSES the fleet secret on these routes —
    // a fallback that kept accepting both would be security theatre.
    process.env.RELAY_RETRACT_SECRET = "retract-only-secret";
    const t = await intelHistoryAppendTest();

    const withFleet = await t.fetch(
      "/relay/intel-history/retract",
      post({ dedupeKeys: ["k"], reason: "r" }, RELAY_SECRET),
    );
    expect(withFleet.status).toBe(401);

    const withRetract = await t.fetch(
      "/relay/intel-history/retract",
      post({ dedupeKeys: ["k"], reason: "r" }, "retract-only-secret"),
    );
    expect(withRetract.status).toBe(200);

    // Ingest is unaffected — the seeders keep appending with the fleet secret.
    const ingest = await t.fetch(
      "/relay/intel-history",
      post(
        {
          domain: "conflict",
          resource: "conflict-events",
          runId: "run-1",
          records: [record({ dedupeKey: "fresh" })],
        },
        RELAY_SECRET,
      ),
    );
    expect(ingest.status).toBe(200);
  });

  test("fails closed instead of falling back to the fleet secret", async () => {
    delete process.env.RELAY_RETRACT_SECRET;
    const t = await intelHistoryAppendTest();
    for (const [path, body] of [
      ["/relay/intel-history/retract", { dedupeKeys: ["k"], reason: "r" }],
      ["/relay/intel-history/restore", { dedupeKeys: ["k"] }],
      ["/relay/intel-history/retractions", {}],
    ] as const) {
      const res = await t.fetch(
        path,
        post(body, RELAY_SECRET),
      );
      expect(res.status, `${path} must require RELAY_RETRACT_SECRET`).toBe(401);
    }
  });

  test("rejects every intel-history route when its credential is unconfigured", async () => {
    // Fail closed: an unconfigured deployment must not accept archive
    // deletions from anyone who guesses the path.
    delete process.env.RELAY_SHARED_SECRET;
    delete process.env.RELAY_RETRACT_SECRET;
    const t = convexTest(schema, modules);

    for (const [path, body] of [
      ["/relay/intel-history/retract", { dedupeKeys: ["k"], reason: "r" }],
      ["/relay/intel-history/restore", { dedupeKeys: ["k"] }],
      ["/relay/intel-history/retractions", {}],
      ["/relay/intel-history", { domain: "conflict", resource: "r", runId: "1", records: [] }],
    ] as const) {
      const res = await t.fetch(path, post(body, ""));
      expect(res.status, `${path} must reject when no secret is configured`).toBe(401);
    }
  });
});