Sync from felladrin/MiniSearch@6c9fb20 (part 2)

server/biEncoderService.ts

@@ -10,9 +10,8 @@ const MODEL_HF_REPO =
 
 /**
  * The ONNX export. ~450 MB, multilingual (50+ languages), 384-dimensional
- * embeddings. Fast enough for batched passage encoding: a batch of 64 passages
- * takes ~30 ms on CPU, well within the 20 s client timeout even for six pages
- * with hundreds of passages each.
+ * embeddings. CPU cost depends on passage length and pool size; the page-content
+ * selector limits dense scoring to 256 passages. See docs/page-content.md.
  */
 const MODEL_HF_FILE = "onnx/model.onnx";
 
@@ -23,8 +22,7 @@ const MODEL_HF_FILE = "onnx/model.onnx";
  */
 const MAX_SEQUENCE_LENGTH = 256;
 
-/** Batch size for passage encoding. Larger batches speed up encoding but use
- * more memory; 64 is a safe default on CPU. */
+/** Maximum concurrent single-text inference calls within each chunk. */
 const BATCH_SIZE = 64;
 
 let isReady = false;

server/handleTokenVerification.test.ts

@@ -44,15 +44,35 @@ function makeResponse() {
   } as unknown as ServerResponse & { end: ReturnType<typeof vi.fn> };
 }
 
-async function verify(url: string, result: TokenVerificationResult) {
+async function verify(
+  url: string,
+  result: TokenVerificationResult,
+  request: { headers?: Record<string, string>; token?: string } = {},
+) {
   verifyMock.mockResolvedValue(result);
   const response = makeResponse();
-  const outcome = await handleTokenVerification("token", response, {
-    url,
-  } as IncomingMessage);
+  const outcome = await handleTokenVerification(
+    request.token ?? "token",
+    response,
+    { url, headers: request.headers ?? {} } as IncomingMessage,
+  );
   return { response, ...outcome };
 }
 
+/** What a browser sends when the address bar or a bookmark opens a URL. */
+const BROWSER_NAVIGATION_ACCEPT =
+  "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8";
+
+function bodyOf(response: ReturnType<typeof makeResponse>): string {
+  return response.end.mock.calls[0][0];
+}
+
+function contentTypeOf(response: ReturnType<typeof makeResponse>): string {
+  const calls = vi.mocked(response.setHeader).mock.calls;
+  const contentType = calls.find(([name]) => name === "Content-Type");
+  return contentType ? String(contentType[1]) : "";
+}
+
 /**
  * The payload is integers all the way down. A per-client dimension would have
  * to arrive either as a new key or as a leaf that stopped being a number, so
@@ -219,4 +239,152 @@ describe("handleTokenVerification", () => {
       rateLimitedBefore + 1,
     );
   });
+
+  describe("when the request looks like a browser navigation", () => {
+    it("answers an invalid token with a page a person can read", async () => {
+      const { shouldContinue, response } = await verify(
+        "/search/text?q=hello&token=stale",
+        REJECTIONS.invalidToken,
+        { headers: { accept: BROWSER_NAVIGATION_ACCEPT } },
+      );
+
+      expect(shouldContinue).toBe(false);
+      expect(response.statusCode).toBe(401);
+      expect(contentTypeOf(response)).toBe("text/html; charset=utf-8");
+
+      const body = bodyOf(response);
+      expect(body.startsWith("<!doctype html>")).toBe(true);
+      expect(body).toContain("rotated its search token");
+      expect(body).toContain("stops working");
+      expect(body).toContain('href="/"');
+      expect(body).not.toContain("<script");
+    });
+
+    it("answers a missing token the same way, keeping its own status", async () => {
+      const { response } = await verify(
+        "/search/text?q=hello",
+        REJECTIONS.missingToken,
+        { headers: { accept: BROWSER_NAVIGATION_ACCEPT } },
+      );
+
+      expect(response.statusCode).toBe(400);
+      expect(contentTypeOf(response)).toBe("text/html; charset=utf-8");
+      expect(bodyOf(response)).toContain('href="/"');
+    });
+
+    it("keeps the JSON body for a rate-limited request", async () => {
+      const { response } = await verify(
+        "/search/text",
+        REJECTIONS.rateLimited,
+        { headers: { accept: BROWSER_NAVIGATION_ACCEPT } },
+      );
+
+      expect(response.statusCode).toBe(429);
+      expect(contentTypeOf(response)).toBe("application/json");
+      expect(bodyOf(response)).toBe(
+        JSON.stringify({ error: "Too many requests." }),
+      );
+    });
+
+    it("is decided by an Accept header that names text/html", async () => {
+      for (const accept of [
+        "text/html",
+        "TEXT/HTML",
+        "text/html; charset=utf-8",
+        "application/json, text/html;q=0.5",
+      ]) {
+        const { response } = await verify(
+          "/search/text",
+          REJECTIONS.invalidToken,
+          { headers: { accept } },
+        );
+
+        expect(contentTypeOf(response), accept).toBe(
+          "text/html; charset=utf-8",
+        );
+      }
+    });
+
+    it("carries no token, sets no cookie and redirects nowhere", async () => {
+      const { response } = await verify(
+        "/search/text?q=hello&token=stale",
+        REJECTIONS.invalidToken,
+        { headers: { accept: BROWSER_NAVIGATION_ACCEPT }, token: "stale" },
+      );
+
+      const headerNames = vi
+        .mocked(response.setHeader)
+        .mock.calls.map(([name]) => String(name).toLowerCase());
+      expect(headerNames).toEqual(["content-type"]);
+      expect(response.statusCode).toBe(401);
+      expect(bodyOf(response)).not.toContain("token=");
+      expect(bodyOf(response)).not.toContain("stale");
+    });
+
+    it("echoes nothing from the request", async () => {
+      const marker = "borogoves";
+      const { response: first } = await verify(
+        `/search/text?q=${marker}&token=${marker}`,
+        REJECTIONS.invalidToken,
+        {
+          headers: {
+            accept: `text/html;${marker}=1`,
+            referer: `https://${marker}.example/`,
+          },
+          token: marker,
+        },
+      );
+      const { response: second } = await verify(
+        "/page-content",
+        REJECTIONS.missingToken,
+        { headers: { accept: "text/html" } },
+      );
+
+      expect(bodyOf(first)).not.toContain(marker);
+      expect(bodyOf(first)).toBe(bodyOf(second));
+    });
+
+    it("counts the rejection the same way as the JSON answer", async () => {
+      const before = getAuthorizationStats();
+
+      await verify("/search/text", REJECTIONS.invalidToken, {
+        headers: { accept: BROWSER_NAVIGATION_ACCEPT },
+      });
+
+      const after = getAuthorizationStats();
+      expect(after.requests).toBe(before.requests + 1);
+      expect(after.authorized).toBe(before.authorized);
+      expect(after.reasons.invalidToken).toBe(before.reasons.invalidToken + 1);
+      expect(after.bySurface.search.rejected).toBe(
+        before.bySurface.search.rejected + 1,
+      );
+    });
+  });
+
+  describe("when the request does not look like a browser navigation", () => {
+    it.each([
+      ["no Accept header", undefined],
+      ["Accept: */*", "*/*"],
+      ["Accept: application/json", "application/json"],
+      ["an Accept list without text/html", "application/json, text/plain, */*"],
+      ["text/html refused with q=0", "text/html;q=0, application/json"],
+      ["text/* only", "text/*"],
+    ])("keeps the JSON body byte for byte with %s", async (_, accept) => {
+      const { shouldContinue, response } = await verify(
+        "/search/text",
+        REJECTIONS.invalidToken,
+        { headers: accept === undefined ? {} : { accept } },
+      );
+
+      expect(shouldContinue).toBe(false);
+      expect(response.statusCode).toBe(401);
+      expect(response.setHeader).toHaveBeenCalledWith(
+        "Content-Type",
+        "application/json",
+      );
+      expect(bodyOf(response)).toBe(
+        JSON.stringify({ error: "Invalid token." }),
+      );
+    });
+  });
 });

server/handleTokenVerification.ts

@@ -5,7 +5,10 @@ import {
   recordAuthorizedRequest,
   recordRejectedRequest,
 } from "./authorizationSinceLastRestart.ts";
-import { verifyTokenAndRateLimit } from "./verifyTokenAndRateLimit.ts";
+import {
+  type RejectionReason,
+  verifyTokenAndRateLimit,
+} from "./verifyTokenAndRateLimit.ts";
 
 const SURFACES: [pathPrefix: string, surface: AuthorizationSurface][] = [
   ["/search", "search"],
@@ -14,6 +17,53 @@ const SURFACES: [pathPrefix: string, surface: AuthorizationSurface][] = [
   ["/inference", "inference"],
 ];
 
+/**
+ * The rejections a person can land on by opening a search URL that was
+ * bookmarked, shared or embedded before this instance rotated its token, or
+ * one that had the token stripped from it on the way. A rate-limited request
+ * is not one of them: the same URL works again a few seconds later, and the
+ * page below would be telling that person the wrong story.
+ */
+const REASONS_WITH_A_PAGE: ReadonlySet<RejectionReason> = new Set([
+  "missingToken",
+  "invalidToken",
+]);
+
+/**
+ * Static on purpose: nothing from the request goes in, so a caller cannot get
+ * its own query, URL or header reflected back in a `text/html` body. The page
+ * links to the root, where the app takes a token through `/api/config` the way
+ * it always does; it carries no token itself and redirects nowhere, because
+ * handing a token to any caller that turned up with a bad one would remove the
+ * CSRF gate (`docs/security.md`, Search Token Lifecycle). No script and no
+ * external asset, so it renders even where the instance is the only reachable
+ * host.
+ */
+const EXPIRED_SEARCH_LINK_PAGE = `<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<meta name="robots" content="noindex">
+<title>This search link has expired</title>
+<style>
+body { margin: 0; padding: 3rem 1.5rem; font: 1rem/1.6 system-ui, sans-serif; color: #1a1a1a; background: #fff; }
+main { max-width: 36rem; margin: 0 auto; }
+h1 { font-size: 1.5rem; line-height: 1.3; }
+a { color: #0b5fff; }
+@media (prefers-color-scheme: dark) { body { color: #ededed; background: #121212; } a { color: #8ab4ff; } }
+</style>
+</head>
+<body>
+<main>
+<h1>This search link has expired</h1>
+<p>This MiniSearch instance rotated its search token. Every search URL carries that token, so a bookmarked, shared or embedded search URL stops working when the token changes.</p>
+<p><a href="/">Open MiniSearch</a> and search again from there. A search started on that page uses the current token.</p>
+</main>
+</body>
+</html>
+`;
+
 /**
  * The endpoint family a request was aimed at. `request.url` carries the query
  * string too, but the match is anchored at the start of it and only the label
@@ -25,6 +75,31 @@ function resolveSurface(request?: IncomingMessage): AuthorizationSurface {
   return match ? match[1] : "other";
 }
 
+/**
+ * Whether the request looks like a browser navigation, which means its
+ * `Accept` header asks for `text/html` by name. A navigation sends
+ * `text/html,application/xhtml+xml,...`, while `fetch` sends only the wildcard
+ * and API clients send `application/json` or nothing, and all of those keep
+ * the JSON body. An entry with `q=0` is a refusal, not a request, so it does
+ * not count either.
+ */
+function asksForHtml(request?: IncomingMessage): boolean {
+  const accept = request?.headers.accept;
+  if (typeof accept !== "string") return false;
+
+  return accept.split(",").some((entry) => {
+    const [mediaType, ...parameters] = entry.split(";");
+    if (mediaType.trim().toLowerCase() !== "text/html") return false;
+
+    return !parameters.some((parameter) => {
+      const [name, value] = parameter.split("=");
+      return (
+        name.trim().toLowerCase() === "q" && Number.parseFloat(value) === 0
+      );
+    });
+  });
+}
+
 export async function handleTokenVerification(
   token: string | null,
   response: ServerResponse,
@@ -41,6 +116,13 @@ export async function handleTokenVerification(
   if (!result.isAuthorized) {
     recordRejectedRequest(surface, result.reason);
     response.statusCode = result.statusCode;
+
+    if (REASONS_WITH_A_PAGE.has(result.reason) && asksForHtml(request)) {
+      response.setHeader("Content-Type", "text/html; charset=utf-8");
+      response.end(EXPIRED_SEARCH_LINK_PAGE);
+      return { shouldContinue: false };
+    }
+
     response.setHeader("Content-Type", "application/json");
     response.end(JSON.stringify({ error: result.error }));
     return { shouldContinue: false };

server/pageContentService.test.ts

@@ -1,6 +1,9 @@
-import { beforeEach, describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import { repository, version } from "../package.json" with { type: "json" };
 
+const scorePassagesMock = vi.hoisted(() => vi.fn());
+vi.mock("./biEncoderService.ts", () => ({ scorePassages: scorePassagesMock }));
+
 const lookupMock = vi.hoisted(() => vi.fn());
 const pdfGetTextMock = vi.hoisted(() => vi.fn());
 const pdfDestroyMock = vi.hoisted(() => vi.fn());
@@ -24,6 +27,10 @@ import {
   splitIntoPassages,
   splitLongPassage,
 } from "./pageContentService";
+import {
+  getPageReadCircuitStats,
+  PageReadHostBreaker,
+} from "./pageReadHostBreaker";
 import { getPageReadStats } from "./pageReadsSinceLastRestart";
 
 const fetchMock = vi.fn();
@@ -54,6 +61,7 @@ ${paragraph("Cats sleep between twelve and sixteen hours a day.")}
 </body></html>`;
 
 beforeEach(() => {
+  scorePassagesMock.mockReset().mockResolvedValue([]);
   fetchMock.mockReset();
   lookupMock.mockReset();
   lookupMock.mockResolvedValue([{ address: "93.184.216.34", family: 4 }]);
@@ -63,6 +71,30 @@ beforeEach(() => {
   pdfDestroyMock.mockResolvedValue(undefined);
 });
 
+/**
+ * Counts what one call to `fetchPageContents` added, by outcome. A fresh
+ * breaker per call unless one is passed, so the refusals a case injects never
+ * add up to a host being skipped in a later one.
+ */
+async function countOutcomes(url: string, breaker = new PageReadHostBreaker()) {
+  const before = getPageReadStats();
+  await fetchPageContents("cats", [url], breaker);
+  const after = getPageReadStats();
+
+  return {
+    read: after.read - before.read,
+    skipped: Object.fromEntries(
+      Object.entries(after.skipped)
+        .map(([outcome, count]) => [
+          outcome,
+          count - before.skipped[outcome as keyof typeof before.skipped],
+        ])
+        .filter(([, count]) => count !== 0),
+    ),
+    bodiesTruncated: after.bodiesTruncated - before.bodiesTruncated,
+  };
+}
+
 describe("extractReadableText", () => {
   it("keeps the article and drops site chrome, scripts and styles", () => {
     const text = extractReadableText(articlePage);
@@ -381,93 +413,81 @@ describe("selectPassages across scripts", () => {
   }
 });
 
-describe("selectPassagesAcrossPages", () => {
-  // Helper to call the internal selector indirectly through fetchPageContents.
-  async function selectFrom(urls: string[], htmls: string[]) {
-    fetchMock.mockReset();
-    for (const html of htmls) {
-      fetchMock.mockResolvedValueOnce(
-        new Response(html, {
-          status: 200,
-          headers: { "content-type": "text/html" },
-        }),
-      );
+describe.each(["lexical", "dense"])(
+  "selectPassagesAcrossPages (%s)",
+  (mode) => {
+    // Helper to call the internal selector indirectly through fetchPageContents.
+    async function selectFrom(urls: string[], htmls: string[]) {
+      if (mode === "dense") {
+        scorePassagesMock.mockImplementation(async (_query, pool: string[]) =>
+          pool.map((_, i) => -i),
+        );
+      }
+      fetchMock.mockReset();
+      for (const html of htmls) {
+        fetchMock.mockResolvedValueOnce(
+          new Response(html, {
+            status: 200,
+            headers: { "content-type": "text/html" },
+          }),
+        );
+      }
+      return fetchPageContents("test query", urls);
     }
-    return fetchPageContents("test query", urls);
-  }
 
-  it("suppresses a syndicated paragraph that appears on multiple pages", async () => {
-    const syndicated =
-      "This paragraph is syndicated across many sites. ".repeat(10);
-    const uniqueA = "Unique content on page A. ".repeat(10);
-    const uniqueB = "Unique content on page B. ".repeat(10);
-
-    const contents = await selectFrom(
-      ["https://example.com/a", "https://example.com/b"],
-      [
-        `<article><p>${uniqueA}</p><p>${syndicated}</p></article>`,
-        `<article><p>${syndicated}</p><p>${uniqueB}</p></article>`,
-      ],
-    );
-
-    // The syndicated paragraph should appear in only one of the two responses.
-    // Text extraction trims trailing whitespace, so match a trimmed marker
-    // rather than the exact paragraph text.
-    const marker = syndicated.trimEnd();
-    const aText = contents.find((c) => c.url.includes("/a"))?.content ?? "";
-    const bText = contents.find((c) => c.url.includes("/b"))?.content ?? "";
-    const count = (aText + bText).split(marker).length - 1;
-    expect(count).toBe(1);
-  });
-
-  it("drops a page whose passages are all near-duplicates of an earlier page", async () => {
-    // Page A has one passage. Page B has one near-duplicate passage
-    // (Jaccard >= 0.9). The new global dedup should suppress B's copy,
-    // leaving B with no selected passages so it vanishes from the response.
-    // The old per-page code would have returned both pages.
-    // Note: with a query that matches no terms, A wins purely on flatten
-    // order (lower global index), not on relevance. The test guards the
-    // dedup drop path, not the ranking order.
-    const base =
-      "word1 word2 word3 word4 word5 word6 word7 word8 word9 word10 ";
-    const passageA = base.repeat(15);
-    const passageB = `${base}word11 `.repeat(15);
-
-    const contents = await selectFrom(
-      ["https://example.com/a", "https://example.com/b"],
-      [
-        `<article><p>${passageA}</p></article>`,
-        `<article><p>${passageB}</p></article>`,
-      ],
-    );
+    it("suppresses a syndicated paragraph that appears on multiple pages", async () => {
+      const syndicated =
+        "This paragraph is syndicated across many sites. ".repeat(10);
+      const uniqueA = "Unique content on page A. ".repeat(10);
+      const uniqueB = "Unique content on page B. ".repeat(10);
+
+      const contents = await selectFrom(
+        ["https://example.com/a", "https://example.com/b"],
+        [
+          `<article><p>${uniqueA}</p><p>${syndicated}</p></article>`,
+          `<article><p>${syndicated}</p><p>${uniqueB}</p></article>`,
+        ],
+      );
 
-    // Only page A should survive; page B's sole passage is a near-duplicate.
-    expect(contents).toHaveLength(1);
-    expect(contents[0].url).toContain("/a");
-  });
-});
+      // The syndicated paragraph should appear in only one of the two responses.
+      // Text extraction trims trailing whitespace, so match a trimmed marker
+      // rather than the exact paragraph text.
+      const marker = syndicated.trimEnd();
+      const aText = contents.find((c) => c.url.includes("/a"))?.content ?? "";
+      const bText = contents.find((c) => c.url.includes("/b"))?.content ?? "";
+      const count = (aText + bText).split(marker).length - 1;
+      expect(count).toBe(1);
+    });
 
-describe("page read counters", () => {
-  /** Counts what one call to `fetchPageContents` added, by outcome. */
-  async function countOutcomes(url: string, query = "cats") {
-    const before = getPageReadStats();
-    await fetchPageContents(query, [url]);
-    const after = getPageReadStats();
+    it("drops a page whose passages are all near-duplicates of an earlier page", async () => {
+      // Page A has one passage. Page B has one near-duplicate passage
+      // (Jaccard >= 0.9). The new global dedup should suppress B's copy,
+      // leaving B with no selected passages so it vanishes from the response.
+      // The old per-page code would have returned both pages.
+      // Note: with a query that matches no terms, A wins purely on flatten
+      // order (lower global index), not on relevance. The test guards the
+      // dedup drop path, not the ranking order.
+      const base =
+        "word1 word2 word3 word4 word5 word6 word7 word8 word9 word10 ";
+      const passageA = base.repeat(15);
+      const passageB = `${base}word11 `.repeat(15);
+
+      const contents = await selectFrom(
+        ["https://example.com/a", "https://example.com/b"],
+        [
+          `<article><p>${passageA}</p></article>`,
+          `<article><p>${passageB}</p></article>`,
+        ],
+      );
 
-    return {
-      read: after.read - before.read,
-      skipped: Object.fromEntries(
-        Object.entries(after.skipped)
-          .map(([outcome, count]) => [
-            outcome,
-            count - before.skipped[outcome as keyof typeof before.skipped],
-          ])
-          .filter(([, count]) => count !== 0),
-      ),
-      bodiesTruncated: after.bodiesTruncated - before.bodiesTruncated,
-    };
-  }
+      // Only page A should survive; page B's sole passage is a near-duplicate.
+      expect(contents).toHaveLength(1);
+      expect(contents[0].url).toContain("/a");
+    });
+  },
+);
 
+describe("page read counters", () => {
   it("counts a page it could read", async () => {
     respondWithHtml(articlePage);
 
@@ -485,10 +505,27 @@ describe("page read counters", () => {
     });
   });
 
-  it("tells an error status apart from a page that is not a document", async () => {
+  it("classifies HTTP error statuses by class and tells them apart from non-documents", async () => {
     fetchMock.mockResolvedValue(new Response("nope", { status: 403 }));
     expect((await countOutcomes("https://example.com/x")).skipped).toEqual({
-      httpError: 1,
+      httpForbidden: 1,
+    });
+
+    fetchMock.mockResolvedValue(new Response("rate limited", { status: 429 }));
+    expect((await countOutcomes("https://example.com/rate")).skipped).toEqual({
+      httpForbidden: 1,
+    });
+
+    fetchMock.mockResolvedValue(new Response("missing", { status: 404 }));
+    expect(
+      (await countOutcomes("https://example.com/missing")).skipped,
+    ).toEqual({
+      httpNotFound: 1,
+    });
+
+    fetchMock.mockResolvedValue(new Response("server error", { status: 500 }));
+    expect((await countOutcomes("https://example.com/err")).skipped).toEqual({
+      httpOtherError: 1,
     });
 
     // A PDF is no longer dropped as notADocument; it goes through the passage
@@ -579,19 +616,222 @@ describe("page read counters", () => {
     expect(stats.excerptKeptRate).toBeLessThan(65);
   });
 
-  it("records no query or URL anywhere in what it reports", async () => {
+  it("records no query, URL or host anywhere in what it reports", async () => {
+    // Refused enough to box the host, so the breaker is holding its name in
+    // memory at the moment the stats are read.
+    const breaker = new PageReadHostBreaker({ failureThreshold: 1 });
+    fetchMock.mockResolvedValue(new Response("nope", { status: 403 }));
+    await fetchPageContents(
+      "a very distinctive private query",
+      ["https://secret.example.com/wall"],
+      breaker,
+    );
     respondWithHtml(articlePage);
-    await fetchPageContents("a very distinctive private query", [
-      "https://secret.example.com/private-page",
-    ]);
+    await fetchPageContents(
+      "a very distinctive private query",
+      ["https://secret.example.com/private-page"],
+      breaker,
+    );
 
-    const reported = JSON.stringify(getPageReadStats());
+    const reported = JSON.stringify({
+      ...getPageReadStats(),
+      ...getPageReadCircuitStats(),
+      circuitOpens: breaker.getOpens(),
+    });
 
     expect(reported).not.toContain("distinctive");
     expect(reported).not.toContain("secret.example.com");
   });
 });
 
+describe("host circuit breaker", () => {
+  // A short window on fake timers, so that "still boxed" holds for as long as
+  // the test says and not for however long the machine takes to reach the
+  // next line.
+  const breakerOptions = {
+    failureThreshold: 3,
+    resetTimeout: 1000,
+    successThreshold: 1,
+  };
+
+  beforeEach(() => {
+    vi.useFakeTimers();
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  function refuse(status: number) {
+    fetchMock.mockResolvedValue(new Response("nope", { status }));
+  }
+
+  async function boxHost(breaker: PageReadHostBreaker, host: string) {
+    refuse(403);
+    for (const page of ["a", "b", "c"]) {
+      await countOutcomes(`https://${host}/${page}`, breaker);
+    }
+  }
+
+  it("skips a host after three refusals in a row, without a request, and counts the skip", async () => {
+    const breaker = new PageReadHostBreaker(breakerOptions);
+    refuse(403);
+
+    for (const page of ["a", "b", "c"]) {
+      expect(
+        (await countOutcomes(`https://wall.example.com/${page}`, breaker))
+          .skipped,
+      ).toEqual({ httpForbidden: 1 });
+    }
+    expect(fetchMock).toHaveBeenCalledTimes(3);
+    expect(breaker.getOpens()).toBe(1);
+
+    // Boxed now: the fourth read is refused here and never reaches the host,
+    // and it is counted, so read plus skipped still adds up to requested.
+    respondWithHtml(articlePage);
+    expect(
+      (await countOutcomes("https://wall.example.com/d", breaker)).skipped,
+    ).toEqual({ skippedByBreaker: 1 });
+    expect(fetchMock).toHaveBeenCalledTimes(3);
+
+    // The box is per host.
+    expect(
+      (await countOutcomes("https://open.example.com/", breaker)).read,
+    ).toBe(1);
+  });
+
+  it("counts a timeout and a dropped connection toward the box, as an HTTP error does", async () => {
+    const breaker = new PageReadHostBreaker(breakerOptions);
+    const timeout = new Error("The operation was aborted due to timeout");
+    timeout.name = "TimeoutError";
+
+    fetchMock.mockRejectedValue(timeout);
+    await countOutcomes("https://slow.example.com/a", breaker);
+    fetchMock.mockRejectedValue(new TypeError("fetch failed"));
+    await countOutcomes("https://slow.example.com/b", breaker);
+    refuse(503);
+    await countOutcomes("https://slow.example.com/c", breaker);
+
+    respondWithHtml(articlePage);
+    expect(
+      (await countOutcomes("https://slow.example.com/d", breaker)).skipped,
+    ).toEqual({ skippedByBreaker: 1 });
+  });
+
+  it("does not count a dead link or a thin page, and either one ends a run of refusals", async () => {
+    const breaker = new PageReadHostBreaker(breakerOptions);
+
+    refuse(403);
+    await countOutcomes("https://mixed.example.com/a", breaker);
+    await countOutcomes("https://mixed.example.com/b", breaker);
+    // The host answered, so it is not refusing: the run starts over.
+    refuse(404);
+    await countOutcomes("https://mixed.example.com/gone", breaker);
+    refuse(403);
+    await countOutcomes("https://mixed.example.com/c", breaker);
+    await countOutcomes("https://mixed.example.com/d", breaker);
+    respondWithHtml("<html><body><p>Accept cookies</p></body></html>");
+    await countOutcomes("https://mixed.example.com/wall", breaker);
+    refuse(403);
+    await countOutcomes("https://mixed.example.com/e", breaker);
+    await countOutcomes("https://mixed.example.com/f", breaker);
+
+    // Six refusals, never three in a row, so every read reached the host.
+    respondWithHtml(articlePage);
+    expect(
+      (await countOutcomes("https://mixed.example.com/g", breaker)).read,
+    ).toBe(1);
+    expect(fetchMock).toHaveBeenCalledTimes(9);
+    expect(breaker.getOpens()).toBe(0);
+  });
+
+  it("lets one probe through once the window has passed, and a success closes the box", async () => {
+    const breaker = new PageReadHostBreaker(breakerOptions);
+    await boxHost(breaker, "wall.example.com");
+    expect(
+      (await countOutcomes("https://wall.example.com/d", breaker)).skipped,
+    ).toEqual({ skippedByBreaker: 1 });
+
+    await vi.advanceTimersByTimeAsync(breakerOptions.resetTimeout + 1);
+    respondWithHtml(articlePage);
+
+    // The probe reaches the host, and its success lets every read through.
+    expect(
+      (await countOutcomes("https://wall.example.com/e", breaker)).read,
+    ).toBe(1);
+    expect(fetchMock).toHaveBeenCalledTimes(4);
+    // One Response object serves every mocked fetch, and the probe consumed
+    // its body, so the next read needs a fresh one.
+    respondWithHtml(articlePage);
+    expect(
+      (await countOutcomes("https://wall.example.com/f", breaker)).read,
+    ).toBe(1);
+    expect(fetchMock).toHaveBeenCalledTimes(5);
+    expect(breaker.getOpens()).toBe(1);
+  });
+
+  it("boxes the host again when the probe is refused", async () => {
+    const breaker = new PageReadHostBreaker(breakerOptions);
+    await boxHost(breaker, "wall.example.com");
+    await vi.advanceTimersByTimeAsync(breakerOptions.resetTimeout + 1);
+
+    expect(
+      (await countOutcomes("https://wall.example.com/probe", breaker)).skipped,
+    ).toEqual({ httpForbidden: 1 });
+    expect(fetchMock).toHaveBeenCalledTimes(4);
+
+    expect(
+      (await countOutcomes("https://wall.example.com/after", breaker)).skipped,
+    ).toEqual({ skippedByBreaker: 1 });
+    expect(fetchMock).toHaveBeenCalledTimes(4);
+    expect(breaker.getOpens()).toBe(2);
+  });
+
+  it("sends one probe when the boxed host appears twice in the same batch", async () => {
+    const breaker = new PageReadHostBreaker(breakerOptions);
+    await boxHost(breaker, "wall.example.com");
+    await vi.advanceTimersByTimeAsync(breakerOptions.resetTimeout + 1);
+    respondWithHtml(articlePage);
+
+    const before = getPageReadStats();
+    const contents = await fetchPageContents(
+      "cats",
+      ["https://wall.example.com/one", "https://wall.example.com/two"],
+      breaker,
+    );
+    const after = getPageReadStats();
+
+    // The reads run in parallel. The first to ask is the probe; the second is
+    // refused rather than sent behind it, which is the double read the box
+    // exists to stop.
+    expect(fetchMock).toHaveBeenCalledTimes(4);
+    expect(contents).toHaveLength(1);
+    expect(after.read - before.read).toBe(1);
+    expect(
+      after.skipped.skippedByBreaker - before.skipped.skippedByBreaker,
+    ).toBe(1);
+  });
+
+  it("stops a redirect chain at a boxed host without asking it", async () => {
+    const breaker = new PageReadHostBreaker(breakerOptions);
+    await boxHost(breaker, "wall.example.com");
+
+    fetchMock.mockResolvedValue(
+      new Response(null, {
+        status: 302,
+        headers: { location: "https://wall.example.com/landing" },
+      }),
+    );
+    expect(
+      (await countOutcomes("https://open.example.com/moved", breaker)).skipped,
+    ).toEqual({ skippedByBreaker: 1 });
+    // One request more: the redirect itself. The host it pointed at was never
+    // asked, and sending a redirect did not count against the host that sent it.
+    expect(fetchMock).toHaveBeenCalledTimes(4);
+    expect(breaker.getOpens()).toBe(1);
+  });
+});
+
 describe("fetchPageContents", () => {
   it("identifies itself with the name, version and repository in package.json", async () => {
     respondWithHtml(articlePage);
@@ -798,3 +1038,117 @@ describe("fetchPageContents", () => {
     expect(chunksSent).toBeLessThan(40);
   });
 });
+
+describe("dense passage fusion", () => {
+  const passages = [
+    `Oceans and ships. ${"Water covers much of the planet. ".repeat(8)}`,
+    `Cats pets sleep. ${"Animals need food and shelter. ".repeat(8)}`,
+    `Cats pets play. ${"Companions enjoy running outdoors. ".repeat(8)}`,
+  ];
+  const url = "https://example.com/animals";
+  const html = `<article>${passages.map((text) => `<p>${text}</p>`).join("")}</article>`;
+  const trimmed = passages.map((text) => text.trim());
+
+  it("does not score an empty pool", async () => {
+    expect(await selectPassages("cats", [], 6000)).toEqual([]);
+    expect(await fetchPageContents("cats", [])).toEqual([]);
+    expect(scorePassagesMock).not.toHaveBeenCalled();
+  });
+
+  it("fuses the production pool by original identity and returns the dense-assisted winner first", async () => {
+    respondWithHtml(html);
+    scorePassagesMock.mockResolvedValue([0.5, 0.1, 0.9]);
+
+    const contents = await fetchPageContents("cats pets", [url]);
+
+    expect(scorePassagesMock).toHaveBeenCalledExactlyOnceWith(
+      "cats pets",
+      trimmed,
+    );
+    expect(contents).toEqual([
+      { url, content: [trimmed[2], trimmed[1], trimmed[0]].join("\n") },
+    ]);
+    expect(await selectPassages("cats pets", trimmed, 6000)).toEqual([
+      trimmed[2],
+      trimmed[1],
+      trimmed[0],
+    ]);
+  });
+
+  it("preserves the exact lexical output when the model is unavailable or fails", async () => {
+    respondWithHtml(html);
+    const expected = [
+      { url, content: [trimmed[1], trimmed[2], trimmed[0]].join("\n") },
+    ];
+    expect(await fetchPageContents("cats pets", [url])).toEqual(expected);
+    const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+    try {
+      scorePassagesMock.mockRejectedValue(
+        new Error("private query, URL and passage"),
+      );
+      respondWithHtml(html);
+      expect(await fetchPageContents("cats pets", [url])).toEqual(expected);
+      expect(warn).toHaveBeenCalledExactlyOnceWith(
+        "Dense passage scoring failed; using lexical ranking.",
+      );
+    } finally {
+      warn.mockRestore();
+    }
+  });
+
+  it("breaks equal fused scores by original index and equal dense scores deterministically", async () => {
+    scorePassagesMock.mockResolvedValue([1, 0]);
+    expect(await selectPassages("cats", ["oceans", "cats"], 100)).toEqual([
+      "oceans",
+      "cats",
+    ]);
+    scorePassagesMock.mockResolvedValue([0.5, 0.5]);
+    expect(await selectPassages("cats", ["oceans", "cats"], 100)).toEqual([
+      "oceans",
+      "cats",
+    ]);
+  });
+
+  it.each([256, 257])(
+    "bounds dense scoring over the combined %i-passage pool",
+    async (count) => {
+      const texts = Array.from(
+        { length: count },
+        (_, i) =>
+          `Passage ${i}. ${"Trees provide shade in warm weather. ".repeat(6)}`,
+      );
+      const halves = [texts.slice(0, 128), texts.slice(128)];
+      const urls = ["https://example.com/a", "https://example.com/b"];
+      const respond = () => {
+        for (const half of halves) {
+          fetchMock.mockResolvedValueOnce(
+            new Response(
+              `<article>${half.map((text) => `<p>${text}</p>`).join("")}</article>`,
+              { headers: { "content-type": "text/html" } },
+            ),
+          );
+        }
+      };
+      respond();
+      const lexical = await fetchPageContents("trees", urls);
+      scorePassagesMock
+        .mockClear()
+        .mockImplementation(async (_query, pool: string[]) =>
+          pool.map((_, i) => i),
+        );
+      respond();
+      const result = await fetchPageContents("trees", urls);
+      if (count === 256) {
+        expect(scorePassagesMock).toHaveBeenCalledExactlyOnceWith(
+          "trees",
+          texts.map((text) => text.trim()),
+        );
+        expect(result).not.toEqual(lexical);
+      } else {
+        expect(scorePassagesMock).not.toHaveBeenCalled();
+        expect(result).toEqual(lexical);
+      }
+      expect(result.every((page) => page.content.length <= 6000)).toBe(true);
+    },
+  );
+});

server/pageContentService.ts

@@ -2,6 +2,10 @@ import { convert as convertHtmlToPlainText } from "html-to-text";
 import { PDFParse } from "pdf-parse";
 import { repository, version } from "../package.json" with { type: "json" };
 import { scorePassages } from "./biEncoderService.ts";
+import {
+  type PageReadHostBreaker,
+  pageReadHostBreaker,
+} from "./pageReadHostBreaker.ts";
 import {
   type PageReadOutcome,
   recordPageRead,
@@ -19,6 +23,8 @@ const MIN_USEFUL_CHARS = 200;
 const MIN_PASSAGE_CHARS = 180;
 const MAX_PASSAGE_CHARS = 1200;
 const OVERLAP_CHARS = 200;
+const MAX_DENSE_PASSAGES = 256;
+const RRF_K = 60;
 
 const appName = repository.url.slice(repository.url.lastIndexOf("/") + 1);
 
@@ -124,6 +130,18 @@ function isRedirect(status: number): boolean {
   );
 }
 
+function classifyHttpError(
+  status: number,
+): "httpForbidden" | "httpNotFound" | "httpOtherError" {
+  if (status === 401 || status === 403 || status === 429) {
+    return "httpForbidden";
+  }
+  if (status === 404 || status === 410) {
+    return "httpNotFound";
+  }
+  return "httpOtherError";
+}
+
 function findCharset(text: string, pattern: RegExp): string | null {
   const match = pattern.exec(text);
   return match ? match[1] : null;
@@ -162,16 +180,88 @@ type DownloadResult =
   | { outcome: "ok"; html?: string; text?: string; bodyTruncated: boolean }
   | { outcome: Exclude<PageReadOutcome, "read" | "tooLittleText"> };
 
+type HopResult = DownloadResult | { outcome: "redirect"; target: string };
+
+/**
+ * One request of a redirect chain: the fetch, and the body unless the host
+ * answered with a redirect. Everything it can fail on comes back as an outcome
+ * rather than an exception, so its caller can settle the host's circuit on it.
+ */
+async function readHop(url: URL, deadline: AbortSignal): Promise<HopResult> {
+  let response: Response;
+  try {
+    response = await fetch(url, {
+      headers: REQUEST_HEADERS,
+      redirect: "manual",
+      signal: deadline,
+    });
+  } catch (error) {
+    return { outcome: isTimeout(error) ? "timedOut" : "failed" };
+  }
+
+  const location = response.headers.get("location");
+  if (isRedirect(response.status) && location) {
+    await response.body?.cancel().catch(() => {});
+    try {
+      return { outcome: "redirect", target: new URL(location, url).toString() };
+    } catch {
+      return { outcome: "failed" };
+    }
+  }
+
+  if (!response.ok) {
+    await response.body?.cancel().catch(() => {});
+    return { outcome: classifyHttpError(response.status) };
+  }
+
+  const contentType = response.headers.get("content-type") ?? "";
+  if (!READABLE_CONTENT_TYPES.test(contentType.trim())) {
+    // PDF is not in READABLE_CONTENT_TYPES but we still want to read it.
+    if (!contentType.trim().toLowerCase().startsWith("application/pdf")) {
+      await response.body?.cancel().catch(() => {});
+      return { outcome: "notADocument" };
+    }
+  }
+
+  try {
+    const { bytes, truncated } = await readCappedBytes(
+      response,
+      MAX_RESPONSE_BYTES,
+    );
+    if (contentType.trim().toLowerCase().startsWith("application/pdf")) {
+      return {
+        outcome: "ok",
+        text: await extractPdfText(bytes),
+        bodyTruncated: truncated,
+      };
+    }
+    return {
+      outcome: "ok",
+      html: decodeDocument(bytes, contentType),
+      bodyTruncated: truncated,
+    };
+  } catch (error) {
+    return { outcome: isTimeout(error) ? "timedOut" : "failed" };
+  }
+}
+
 /**
  * Follows redirects by hand so that every hop is validated: `redirect:
  * "follow"` would let a public URL bounce the server into a private address.
  * The whole chain shares one deadline, so a page cannot buy extra time by
  * redirecting.
  *
+ * Each hop runs under the circuit of the host it is about to reach, keyed on
+ * `url.host` once `resolvePublicUrl` has cleared it, so a chain that lands on
+ * a boxed host stops there and a redirect counts for the host that sent it.
+ *
  * Returns why it gave up rather than throwing, because the caller counts the
  * reasons and a thrown `Error` would have to be identified by its message.
  */
-async function downloadDocument(rawUrl: string): Promise<DownloadResult> {
+async function downloadDocument(
+  rawUrl: string,
+  breaker: PageReadHostBreaker,
+): Promise<DownloadResult> {
   let target = rawUrl;
   const deadline = AbortSignal.timeout(REQUEST_TIMEOUT_MS);
 
@@ -183,62 +273,19 @@ async function downloadDocument(rawUrl: string): Promise<DownloadResult> {
       return { outcome: "blocked" };
     }
 
-    let response: Response;
-    try {
-      response = await fetch(url, {
-        headers: REQUEST_HEADERS,
-        redirect: "manual",
-        signal: deadline,
-      });
-    } catch (error) {
-      return { outcome: isTimeout(error) ? "timedOut" : "failed" };
-    }
-
-    const location = response.headers.get("location");
-    if (isRedirect(response.status) && location) {
-      await response.body?.cancel().catch(() => {});
-      try {
-        target = new URL(location, url).toString();
-      } catch {
-        return { outcome: "failed" };
-      }
-      continue;
-    }
-
-    if (!response.ok) {
-      await response.body?.cancel().catch(() => {});
-      return { outcome: "httpError" };
-    }
-
-    const contentType = response.headers.get("content-type") ?? "";
-    if (!READABLE_CONTENT_TYPES.test(contentType.trim())) {
-      // PDF is not in READABLE_CONTENT_TYPES but we still want to read it.
-      if (!contentType.trim().toLowerCase().startsWith("application/pdf")) {
-        await response.body?.cancel().catch(() => {});
-        return { outcome: "notADocument" };
-      }
-    }
+    if (!breaker.admit(url.host)) return { outcome: "skippedByBreaker" };
 
+    // Settled whatever happens, or a half-open host would wait forever for a
+    // probe that never reported back.
+    let result: HopResult = { outcome: "failed" };
     try {
-      const { bytes, truncated } = await readCappedBytes(
-        response,
-        MAX_RESPONSE_BYTES,
-      );
-      if (contentType.trim().toLowerCase().startsWith("application/pdf")) {
-        return {
-          outcome: "ok",
-          text: await extractPdfText(bytes),
-          bodyTruncated: truncated,
-        };
-      }
-      return {
-        outcome: "ok",
-        html: decodeDocument(bytes, contentType),
-        bodyTruncated: truncated,
-      };
-    } catch (error) {
-      return { outcome: isTimeout(error) ? "timedOut" : "failed" };
+      result = await readHop(url, deadline);
+    } finally {
+      breaker.settle(url.host, result.outcome);
     }
+
+    if (result.outcome !== "redirect") return result;
+    target = result.target;
   }
 
   return { outcome: "redirectLimit" };
@@ -521,6 +568,43 @@ function rankPassages(
     .sort((a, b) => b.score - a.score || a.index - b.index);
 }
 
+async function rankPassagesWithDenseScores(
+  query: string,
+  passages: { url: string; text: string; positionInPage: number }[],
+): Promise<RankedPassage[]> {
+  const lexicalRanked = rankPassages(query, passages);
+  if (passages.length === 0 || passages.length > MAX_DENSE_PASSAGES) {
+    return lexicalRanked;
+  }
+
+  let denseScores: number[];
+  try {
+    denseScores = await scorePassages(
+      query,
+      passages.map((p) => p.text),
+    );
+  } catch {
+    console.warn("Dense passage scoring failed; using lexical ranking.");
+    return lexicalRanked;
+  }
+  if (denseScores.length === 0) return lexicalRanked;
+
+  const denseRanks: number[] = [];
+  denseScores
+    .map((score, index) => ({ score, index }))
+    .sort((a, b) => b.score - a.score || a.index - b.index)
+    .forEach(({ index }, rank) => {
+      denseRanks[index] = rank + 1;
+    });
+
+  return lexicalRanked
+    .map((passage, rank) => ({
+      ...passage,
+      score: 1 / (RRF_K + rank + 1) + 1 / (RRF_K + denseRanks[passage.index]),
+    }))
+    .sort((a, b) => b.score - a.score || a.index - b.index);
+}
+
 const JACCARD_THRESHOLD = 0.9;
 
 /**
@@ -602,7 +686,7 @@ function selectPassagesAcrossPages(
  * what was transferred and never what the model read.
  *
  * Lexical and dense (bi-encoder) scores are fused with reciprocal rank fusion;
- * when the model is unavailable the lexical score alone is used.
+ * unavailable or failed scoring and pools above 256 passages keep lexical order.
  *
  * @param passages - Candidate passages from a single page; the wrapper feeds
  * them through the production selector so tests exercise the real code path.
@@ -620,34 +704,15 @@ export async function selectPassages(
     positionInPage: index,
   }));
 
-  const lexicalRanked = rankPassages(query, input);
-
-  // Dense scores from the bi-encoder; empty array when the model is not loaded.
-  const denseScores = await scorePassages(query, passages);
-
-  // Reciprocal rank fusion: combine lexical and dense ranks.
-  const RRF_K = 60;
-  const fused = lexicalRanked.map((p, i) => {
-    const lexicalRank = i + 1;
-    const denseRank =
-      denseScores.length > 0
-        ? [...denseScores]
-            .map((s, idx) => ({ s, idx }))
-            .sort((a, b) => b.s - a.s)
-            .findIndex((r) => r.idx === i) + 1
-        : lexicalRank;
-    return {
-      ...p,
-      score:
-        1 / (RRF_K + lexicalRank) +
-        (denseScores.length > 0 ? 1 / (RRF_K + denseRank) : 0),
-    };
-  });
+  const fused = await rankPassagesWithDenseScores(query, input);
 
   return selectPassagesAcrossPages(fused, maxChars, maxChars).get("") ?? [];
 }
 
-async function fetchPageContent(url: string): Promise<{
+async function fetchPageContent(
+  url: string,
+  breaker: PageReadHostBreaker,
+): Promise<{
   url: string;
   passages: string[];
   durationMs: number;
@@ -656,7 +721,7 @@ async function fetchPageContent(url: string): Promise<{
   const startedAt = performance.now();
   const since = () => performance.now() - startedAt;
 
-  const download = await downloadDocument(url);
+  const download = await downloadDocument(url, breaker);
 
   if (download.outcome !== "ok") {
     recordPageRead({ outcome: download.outcome, durationMs: since() });
@@ -702,18 +767,25 @@ async function fetchPageContent(url: string): Promise<{
  * character cap and near-duplicate suppression, so syndicated paragraphs that
  * appear on several pages reach the prompt once.
  *
+ * A host that refused enough recent reads is skipped without a request, under
+ * `breaker`, and that skip is counted like any other outcome.
+ *
  * Nothing here is logged. Every read is counted instead, by outcome, in
  * `pageReadsSinceLastRestart`, since a line naming the query or the URL would
  * record what someone searched for.
  *
  * @param urls - Page URLs to read, already ranked by the search pipeline
+ * @param breaker - The per-host circuits to read under; injectable for tests
  * @returns One entry per page that yielded usable text
  */
 export async function fetchPageContents(
   query: string,
   urls: string[],
+  breaker: PageReadHostBreaker = pageReadHostBreaker,
 ): Promise<PageContent[]> {
-  const results = await Promise.all(urls.map((url) => fetchPageContent(url)));
+  const results = await Promise.all(
+    urls.map((url) => fetchPageContent(url, breaker)),
+  );
 
   const pages = results.filter((r): r is FetchedPage => r !== null);
 
@@ -727,7 +799,7 @@ export async function fetchPageContents(
     })),
   );
 
-  const ranked = rankPassages(query, allPassages);
+  const ranked = await rankPassagesWithDenseScores(query, allPassages);
   const selectedByUrl = selectPassagesAcrossPages(
     ranked,
     MAX_PAGE_CHARS * pages.length,

server/pageReadHostBreaker.ts

@@ -0,0 +1,111 @@
+import type { PageReadOutcome } from "./pageReadsSinceLastRestart.ts";
+import {
+  CircuitBreaker,
+  type CircuitBreakerOptions,
+} from "./utils/circuitBreaker.ts";
+
+/**
+ * The outcomes that are the host's doing, and so count toward boxing it: a
+ * bot wall or rate limit (`httpForbidden`), any other non-ok status, a read
+ * that hit the deadline, and a connection that was refused or dropped. A dead
+ * link (`httpNotFound`) is the host answering, a page with too little text is
+ * the page's fault, and `blocked` never reached the host at all, so none of
+ * those count, and each of them resets the run of refusals the way a read
+ * that worked does.
+ */
+const HOST_REFUSALS: ReadonlySet<string> = new Set<PageReadOutcome>([
+  "httpForbidden",
+  "httpOtherError",
+  "timedOut",
+  "failed",
+]);
+
+/**
+ * One circuit per host in front of the page reader, in the shape of the
+ * SearXNG breaker: after `failureThreshold` refusals in a row a host is
+ * skipped for `resetTimeout`, then one read is let through to probe it, and a
+ * success closes the circuit again. Unlike the SearXNG breaker it sits in
+ * front of code that reports its failures as return values rather than
+ * exceptions, so it drives the circuit through `admit` and `settle` instead
+ * of `execute`.
+ *
+ * The host names it keys on stay in this object and never reach `/status`,
+ * the log, or a counter: a host comes from a search result, so naming it
+ * would say something about what someone searched for. What leaves here is
+ * how often a circuit opened.
+ */
+export class PageReadHostBreaker {
+  private readonly circuits: CircuitBreaker;
+  /**
+   * Hosts with a half-open probe in flight. Reads run in parallel over a
+   * batch, and the class admits every caller while a circuit is half-open,
+   * so without this a host that appears twice in the batch would be probed
+   * twice, which is the double read the breaker exists to stop.
+   */
+  private readonly probing = new Set<string>();
+  private opens = 0;
+
+  constructor(options: Partial<CircuitBreakerOptions> = {}) {
+    this.circuits = new CircuitBreaker(options);
+  }
+
+  /**
+   * Whether a read from `host` may go ahead now. Every admitted read must be
+   * settled, or a half-open host keeps waiting for a probe that never ends.
+   */
+  admit(host: string): boolean {
+    if (!this.circuits.allows(host)) return false;
+
+    if (this.circuits.getState(host) === "HALF_OPEN") {
+      if (this.probing.has(host)) return false;
+      this.probing.add(host);
+    }
+
+    return true;
+  }
+
+  /** Records how an admitted read from `host` ended. */
+  settle(host: string, outcome: string): void {
+    this.probing.delete(host);
+
+    if (!HOST_REFUSALS.has(outcome)) {
+      this.circuits.recordSuccess(host);
+      return;
+    }
+
+    const opensBefore = this.circuits.getOpens(host);
+    this.circuits.recordFailure(host);
+    this.opens += this.circuits.getOpens(host) - opensBefore;
+  }
+
+  /** How often any host's circuit has opened, which names no host. */
+  getOpens(): number {
+    return this.opens;
+  }
+}
+
+/**
+ * Three refusals rather than the SearXNG breaker's five: a refused read costs
+ * the batch its whole wait, up to the 6 s deadline, and on the public instance
+ * the hosts that refuse do so on every read, so the third is as sure as the
+ * fifth. Five minutes rather than one: at that instance's rate the same host
+ * comes back minutes apart, within one user's run of searches, and a minute's
+ * box would usually have expired by then. One healthy probe closes it again,
+ * as it does for SearXNG.
+ */
+export const pageReadHostBreaker = new PageReadHostBreaker({
+  failureThreshold: 3,
+  resetTimeout: 300_000,
+  successThreshold: 1,
+});
+
+/**
+ * How often a host's circuit has opened, for `/status`, beside the skipped
+ * reads it counts under `pageReads.skipped.skippedByBreaker`: the skips
+ * measure what the box saved, the opens are what `failureThreshold` moves.
+ */
+export function getPageReadCircuitStats() {
+  return {
+    circuitOpens: pageReadHostBreaker.getOpens(),
+  };
+}

server/pageReadsSinceLastRestart.test.ts

@@ -37,20 +37,26 @@ describe("recordPageRead", () => {
   it("counts every outcome against the total requested", () => {
     const counted = delta(() => {
       recordPageRead({ outcome: "blocked", durationMs: 1 });
+      recordPageRead({ outcome: "skippedByBreaker", durationMs: 0 });
       recordPageRead({ outcome: "notADocument", durationMs: 2 });
-      recordPageRead({ outcome: "httpError", durationMs: 3 });
-      recordPageRead({ outcome: "redirectLimit", durationMs: 4 });
+      recordPageRead({ outcome: "httpForbidden", durationMs: 3 });
+      recordPageRead({ outcome: "httpNotFound", durationMs: 4 });
+      recordPageRead({ outcome: "httpOtherError", durationMs: 5 });
+      recordPageRead({ outcome: "redirectLimit", durationMs: 6 });
       recordPageRead({ outcome: "timedOut", durationMs: 6000 });
-      recordPageRead({ outcome: "tooLittleText", durationMs: 5 });
-      recordPageRead({ outcome: "failed", durationMs: 6 });
+      recordPageRead({ outcome: "tooLittleText", durationMs: 7 });
+      recordPageRead({ outcome: "failed", durationMs: 8 });
     });
 
-    expect(counted.requested).toBe(7);
+    expect(counted.requested).toBe(10);
     expect(counted.read).toBe(0);
     expect(counted.skipped).toEqual({
       blocked: 1,
+      skippedByBreaker: 1,
       notADocument: 1,
-      httpError: 1,
+      httpForbidden: 1,
+      httpNotFound: 1,
+      httpOtherError: 1,
       redirectLimit: 1,
       timedOut: 1,
       tooLittleText: 1,

server/pageReadsSinceLastRestart.ts

@@ -1,12 +1,17 @@
 /**
  * How a single page read ended. Everything other than `read` means the page
  * contributed nothing to the prompt and its result kept only its snippet.
+ * `skippedByBreaker` is a read that never left the server: its host had
+ * refused enough recent reads that `pageReadHostBreaker` boxed it.
  */
 export type PageReadOutcome =
   | "read"
   | "blocked"
+  | "skippedByBreaker"
   | "notADocument"
-  | "httpError"
+  | "httpForbidden"
+  | "httpNotFound"
+  | "httpOtherError"
   | "redirectLimit"
   | "timedOut"
   | "tooLittleText"
@@ -18,15 +23,19 @@ export type PageReadOutcome =
  * Deliberately counts and nothing else. The obvious thing to log while reading
  * pages is the query and the URL, and that is exactly the pair that says what
  * a user searched for, so none of it is recorded: no query, no URL, no host, no
- * timestamp per read. What is left still answers the questions worth asking of
- * this feature, because every counter below is attached to a constant someone
- * will want to move.
+ * timestamp per read. The hosts the breaker keys on stay inside it and reach
+ * this file only as the count of reads it skipped. What is left still answers
+ * the questions worth asking of this feature, because every counter below is
+ * attached to a constant someone will want to move.
  */
 const outcomes: Record<PageReadOutcome, number> = {
   read: 0,
   blocked: 0,
+  skippedByBreaker: 0,
   notADocument: 0,
-  httpError: 0,
+  httpForbidden: 0,
+  httpNotFound: 0,
+  httpOtherError: 0,
   redirectLimit: 0,
   timedOut: 0,
   tooLittleText: 0,

server/rankSearchResults.test.ts

@@ -12,12 +12,40 @@ beforeEach(() => {
 });
 
 describe("rankSearchResults", () => {
+  it.each([
+    ["text", false],
+    ["images", true],
+  ] as const)(
+    "records %s independently of preserveTopResults=%s",
+    async (searchType, preserveTopResults) => {
+      const { getRerankingStats } = await import("./rerankingSinceLastRestart");
+      const { rankSearchResults } = await import("./rankSearchResults");
+      const before = getRerankingStats();
+      const otherType = searchType === "text" ? "images" : "text";
+      mockRerank.mockResolvedValue([{ index: 0, relevance_score: 1 }]);
+
+      await rankSearchResults(
+        "query",
+        searchType,
+        [["Title", "Snippet", "https://example.com"]],
+        preserveTopResults,
+      );
+
+      expect(getRerankingStats().byType[searchType].reranks).toBe(
+        before.byType[searchType].reranks + 1,
+      );
+      expect(getRerankingStats().byType[otherType]).toEqual(
+        before.byType[otherType],
+      );
+    },
+  );
+
   it("should return empty array when no results provided", async () => {
     mockRerank.mockResolvedValue(
       [] as { index: number; relevance_score: number }[],
     );
     const { rankSearchResults } = await import("./rankSearchResults");
-    const result = await rankSearchResults("test query", []);
+    const result = await rankSearchResults("test query", "text", []);
     expect(result).toEqual([]);
     expect(mockRerank).toHaveBeenCalledWith("test query", []);
   });
@@ -28,7 +56,7 @@ describe("rankSearchResults", () => {
       { index: 1, relevance_score: 0.5 },
     ] as { index: number; relevance_score: number }[]);
     const { rankSearchResults } = await import("./rankSearchResults");
-    await rankSearchResults("Test Query", [
+    await rankSearchResults("Test Query", "text", [
       ["Title A", "Content A", "https://a.com"],
       ["Title B", "Content B", "https://b.com"],
     ]);
@@ -42,16 +70,18 @@ describe("rankSearchResults", () => {
 
   it("should sort results by score descending when preserveTopResults is false", async () => {
     mockRerank.mockResolvedValue([
-      { index: 0, relevance_score: 0.9 },
-      { index: 1, relevance_score: 0.8 },
+      { index: 0, relevance_score: 0.8 },
+      { index: 1, relevance_score: 0.9 },
+      { index: 2, relevance_score: 0.5 },
     ] as { index: number; relevance_score: number }[]);
     const { rankSearchResults } = await import("./rankSearchResults");
-    const result = await rankSearchResults("query", [
+    const result = await rankSearchResults("query", "text", [
       ["A", "a", "https://a.com"],
       ["B", "b", "https://b.com"],
+      ["C", "c", "https://c.com"],
     ]);
-    // Results are sorted by score descending (A has higher score)
-    expect(result[0][0]).toBe("A");
+    // Results are sorted by score descending (B has higher score than A, C is dropped by score filter)
+    expect(result[0][0]).toBe("B");
   });
 
   it("carries the reranker score through as a fourth tuple element", async () => {
@@ -61,7 +91,7 @@ describe("rankSearchResults", () => {
       { index: 2, relevance_score: 8.8 },
     ] as { index: number; relevance_score: number }[]);
     const { rankSearchResults } = await import("./rankSearchResults");
-    const result = await rankSearchResults("query", [
+    const result = await rankSearchResults("query", "text", [
       ["A", "a", "https://a.com"],
       ["B", "b", "https://b.com"],
       ["C", "c", "https://c.com"],
@@ -83,6 +113,7 @@ describe("rankSearchResults", () => {
     const { rankSearchResults } = await import("./rankSearchResults");
     const result = await rankSearchResults(
       "query",
+      "text",
       [
         ["Top", "top", "https://top.com"],
         ["A", "a", "https://a.com"],
@@ -112,6 +143,7 @@ describe("rankSearchResults", () => {
     const { rankSearchResults } = await import("./rankSearchResults");
     const result = await rankSearchResults(
       "query",
+      "text",
       [
         ["Top", "top content", "https://top.com"],
         ["Other", "other content", "https://other.com"],
@@ -126,7 +158,7 @@ describe("rankSearchResults", () => {
       [] as { index: number; relevance_score: number }[],
     );
     const { rankSearchResults } = await import("./rankSearchResults");
-    const result = await rankSearchResults("query", [
+    const result = await rankSearchResults("query", "text", [
       ["A", "a", "https://a.com"],
     ]);
     expect(result).toEqual([]);
@@ -139,7 +171,9 @@ describe("rankSearchResults", () => {
     }[]);
     const { rankSearchResults } = await import("./rankSearchResults");
     const longTitle = "A".repeat(600);
-    await rankSearchResults("query", [[longTitle, "short", "https://a.com"]]);
+    await rankSearchResults("query", "text", [
+      [longTitle, "short", "https://a.com"],
+    ]);
     const docs = mockRerank.mock.calls[0][1] as string[];
     // Truncation is now token-based inside the reranker, so the full document
     // reaches it; nothing is cut by character count upstream.
@@ -153,7 +187,7 @@ describe("rankSearchResults", () => {
       relevance_score: number;
     }[]);
     const { rankSearchResults } = await import("./rankSearchResults");
-    await rankSearchResults("query", [
+    await rankSearchResults("query", "text", [
       ["Title", 'Content with "quotes"', "https://a.com"],
     ]);
     const docs = mockRerank.mock.calls[0][1] as string[];
@@ -173,8 +207,8 @@ describe("rankSearchResults", () => {
     ] as { index: number; relevance_score: number }[]);
 
     const before = getRerankingStats();
-    await rankSearchResults("test", results);
-    await rankSearchResults("test", results, true);
+    await rankSearchResults("test", "text", results);
+    await rankSearchResults("test", "text", results, true);
 
     expect(getRerankingStats().reranks).toBe(before.reranks + 2);
 
@@ -183,7 +217,7 @@ describe("rankSearchResults", () => {
     mockRerank.mockResolvedValue(
       [] as { index: number; relevance_score: number }[],
     );
-    await rankSearchResults("test", []);
+    await rankSearchResults("test", "text", []);
 
     expect(getRerankingStats().reranks).toBe(before.reranks + 2);
   });
@@ -198,7 +232,7 @@ describe("rankSearchResults", () => {
     ] as { index: number; relevance_score: number }[]);
 
     const before = getRerankingStats();
-    const ranked = await rankSearchResults("test", [
+    const ranked = await rankSearchResults("test", "text", [
       ["A", "a", "https://a.com"],
       ["B", "b", "https://b.com"],
       ["C", "c", "https://c.com"],
@@ -221,7 +255,7 @@ describe("rankSearchResults", () => {
     ] as { index: number; relevance_score: number }[]);
     const { rankSearchResults } = await import("./rankSearchResults");
 
-    const ranked = await rankSearchResults("test", [
+    const ranked = await rankSearchResults("test", "text", [
       ["A", "a", "https://a.com"],
       ["B", "b", "https://b.com"],
       ["C", "c", "https://c.com"],
@@ -252,6 +286,7 @@ describe("rankSearchResults", () => {
 
     await rankSearchResults(
       "test",
+      "text",
       Array.from({ length: 10 }, (_unused, index) => [
         `T${index}`,
         `C${index}`,

server/rankSearchResults.ts

@@ -1,8 +1,11 @@
 import { rerank } from "./rerankerService.ts";
 import { recordRerank } from "./rerankingSinceLastRestart.ts";
 
+import type { SearchType } from "./searchesSinceLastRestart.ts";
+
 export async function rankSearchResults(
   query: string,
+  searchType: SearchType,
   searchResults: [title: string, content: string, url: string][],
   preserveTopResults = false,
 ): Promise<ScoredSearchResultTuple[]> {
@@ -22,7 +25,7 @@ export async function rankSearchResults(
   // instances where searches are coming back empty.
   const report = (considered: number, kept: number) => {
     if (considered === 0) return;
-    recordRerank({
+    recordRerank(searchType, {
       considered,
       kept,
       durationMs: performance.now() - startedAt,

server/rerankingSinceLastRestart.test.ts

@@ -0,0 +1,66 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+beforeEach(() => vi.resetModules());
+
+describe("reranking counters", () => {
+  it("weights totals by reranks and results, retaining independent snapshots", async () => {
+    const { recordRerank, getRerankingStats } = await import(
+      "./rerankingSinceLastRestart.ts"
+    );
+    recordRerank("text", {
+      considered: 10,
+      kept: 2,
+      durationMs: 100.4,
+      usedPercentageFallback: false,
+    });
+    const first = getRerankingStats();
+    recordRerank("text", {
+      considered: 10,
+      kept: 3,
+      durationMs: 300.4,
+      usedPercentageFallback: true,
+    });
+    recordRerank("images", {
+      considered: 2,
+      kept: 2,
+      durationMs: 20.4,
+      usedPercentageFallback: false,
+    });
+
+    expect(getRerankingStats()).toEqual({
+      reranks: 3,
+      averageMs: 140,
+      considered: 22,
+      kept: 7,
+      keptRate: 31.8,
+      fallbackApplied: 1,
+      skippedUnhealthy: 0,
+      failed: 0,
+      byType: {
+        text: {
+          reranks: 2,
+          averageMs: 200,
+          considered: 20,
+          kept: 5,
+          keptRate: 25,
+        },
+        images: {
+          reranks: 1,
+          averageMs: 20,
+          considered: 2,
+          kept: 2,
+          keptRate: 100,
+        },
+      },
+    });
+    expect(first.byType.text).toEqual({
+      reranks: 1,
+      averageMs: 100,
+      considered: 10,
+      kept: 2,
+      keptRate: 20,
+    });
+    first.byType.images.reranks = 99;
+    expect(getRerankingStats().byType.images.reranks).toBe(1);
+  });
+});

server/rerankingSinceLastRestart.ts

@@ -7,29 +7,42 @@
  * two thresholds should be moved against, and today they have none.
  */
 
-let reranks = 0;
-let totalMs = 0;
-let totalConsidered = 0;
-let totalKept = 0;
+import type { SearchType } from "./searchesSinceLastRestart.ts";
+
+interface RerankCounts {
+  reranks: number;
+  totalMs: number;
+  considered: number;
+  kept: number;
+}
+
+const byType: Record<SearchType, RerankCounts> = {
+  text: { reranks: 0, totalMs: 0, considered: 0, kept: 0 },
+  images: { reranks: 0, totalMs: 0, considered: 0, kept: 0 },
+};
 let fallbackApplied = 0;
 let skippedUnhealthy = 0;
 let failed = 0;
 
-export function recordRerank({
-  considered,
-  kept,
-  durationMs,
-  usedPercentageFallback,
-}: {
-  considered: number;
-  kept: number;
-  durationMs: number;
-  usedPercentageFallback: boolean;
-}): void {
-  reranks++;
-  totalMs += durationMs;
-  totalConsidered += considered;
-  totalKept += kept;
+export function recordRerank(
+  searchType: SearchType,
+  {
+    considered,
+    kept,
+    durationMs,
+    usedPercentageFallback,
+  }: {
+    considered: number;
+    kept: number;
+    durationMs: number;
+    usedPercentageFallback: boolean;
+  },
+): void {
+  const counts = byType[searchType];
+  counts.reranks++;
+  counts.totalMs += durationMs;
+  counts.considered += considered;
+  counts.kept += kept;
   if (usedPercentageFallback) fallbackApplied++;
 }
 
@@ -48,15 +61,34 @@ export function recordRerankFailed(): void {
  * `pageReads.excerptKeptRate` is: a count of searches whose results were
  * filtered reads ~100% whatever the threshold is, and settles nothing.
  */
-export function getRerankingStats() {
+function summarize({ reranks, totalMs, considered, kept }: RerankCounts) {
   return {
     reranks,
     averageMs: Math.round(totalMs / reranks || 0),
-    considered: totalConsidered,
-    kept: totalKept,
-    keptRate: Number(((totalKept / totalConsidered) * 100 || 0).toFixed(1)),
+    considered,
+    kept,
+    keptRate: Number(((kept / considered) * 100 || 0).toFixed(1)),
+  };
+}
+
+export function getRerankingStats() {
+  const totals = Object.values(byType).reduce(
+    (total, counts) => ({
+      reranks: total.reranks + counts.reranks,
+      totalMs: total.totalMs + counts.totalMs,
+      considered: total.considered + counts.considered,
+      kept: total.kept + counts.kept,
+    }),
+    { reranks: 0, totalMs: 0, considered: 0, kept: 0 },
+  );
+  return {
+    ...summarize(totals),
     fallbackApplied,
     skippedUnhealthy,
     failed,
+    byType: {
+      text: summarize(byType.text),
+      images: summarize(byType.images),
+    },
   };
 }

server/rerankingStatus.test.ts

@@ -0,0 +1,222 @@
+import {
+  createServer,
+  type IncomingMessage,
+  type ServerResponse,
+} from "node:http";
+import type { AddressInfo } from "node:net";
+import type { PreviewServer } from "vite";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const model = vi.hoisted(() => ({
+  healthy: true,
+  durationMs: 0,
+  now: 0,
+  scores: [1, 9, 8.9, 8.8],
+  fail: false,
+}));
+
+vi.mock("./rerankerService.ts", () => ({
+  getRerankerStatus: async () => model.healthy,
+  rerank: async (_query: string, documents: string[]) => {
+    if (model.fail) throw new Error("fixture model failure");
+    model.now += model.durationMs;
+    return documents.map((_document, index) => ({
+      index,
+      relevance_score: model.scores[index],
+    }));
+  },
+}));
+vi.mock("./biEncoderService.ts", () => ({
+  getBiEncoderStatus: async () => true,
+}));
+vi.mock("./webSearchService.ts", async (importOriginal) => ({
+  ...(await importOriginal<typeof import("./webSearchService.ts")>()),
+  fetchSearXNG: vi.fn(),
+  getWebSearchServiceStatus: async () => "healthy",
+}));
+
+type Handler = (
+  request: IncomingMessage,
+  response: ServerResponse,
+  next: () => void,
+) => Promise<void>;
+const textResults = [
+  ["Pinned", "private snippet", "https://example.com/0"],
+  ["High", "private snippet", "https://example.com/1"],
+  ["Middle", "private snippet", "https://example.com/2"],
+  ["Low", "private snippet", "https://example.com/3"],
+] as [string, string, string][];
+const imageResults = textResults.map(
+  ([title, , url]) =>
+    [title, `${url}.jpg`, `${url}/thumbnail`, url] as [
+      string,
+      string,
+      string,
+      string,
+    ],
+);
+const emptyStats = {
+  reranks: 0,
+  averageMs: 0,
+  considered: 0,
+  kept: 0,
+  keptRate: 0,
+};
+let server: ReturnType<typeof createServer>;
+let baseUrl: string;
+
+async function request(path: string) {
+  const response = await fetch(`${baseUrl}${path}`);
+  expect(response.status).toBe(200);
+  return response.json();
+}
+
+beforeEach(async () => {
+  vi.resetModules();
+  model.healthy = true;
+  model.durationMs = 0;
+  model.now = 0;
+  model.fail = false;
+  model.scores = [1, 9, 8.9, 8.8];
+  vi.spyOn(performance, "now").mockImplementation(() => model.now);
+  const { fetchSearXNG } = await import("./webSearchService.ts");
+  vi.mocked(fetchSearXNG).mockImplementation(async (_query, type) =>
+    type === "text" ? textResults : imageResults,
+  );
+  const { addVerifiedToken } = await import("./verifiedTokens.ts");
+  addVerifiedToken("fixture-session");
+  const { searchEndpointServerHook } = await import(
+    "./searchEndpointServerHook.ts"
+  );
+  const { statusEndpointServerHook } = await import(
+    "./statusEndpointServerHook.ts"
+  );
+  const handlers: Handler[] = [];
+  const host = {
+    middlewares: { use: (handler: Handler) => handlers.push(handler) },
+    config: { define: { VITE_BUILD_DATE_TIME: "2026-09-18T00:00:00Z" } },
+  } as unknown as PreviewServer;
+  searchEndpointServerHook(host);
+  statusEndpointServerHook(host);
+  server = createServer((req, res) => {
+    let index = 0;
+    const next = () => {
+      const handler = handlers[index++];
+      if (!handler) {
+        res.writeHead(404).end();
+        return;
+      }
+      handler(req, res, next).catch(() => res.writeHead(500).end());
+    };
+    next();
+  });
+  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+  baseUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
+});
+
+afterEach(async () => {
+  vi.restoreAllMocks();
+  if (server)
+    await new Promise<void>((resolve, reject) =>
+      server.close((error) => (error ? reject(error) : resolve())),
+    );
+});
+
+describe("search reranking counters through HTTP status", () => {
+  it("keeps text and images separate while preserving combined totals and result order", async () => {
+    model.durationMs = 100;
+    expect(
+      await request("/search/text?q=private-query&token=fixture-session"),
+    ).toEqual([
+      [...textResults[0], 1],
+      [...textResults[1], 9],
+      [...textResults[2], 8.9],
+    ]);
+    const first = (await request("/status")).reranker;
+    expect(first.byType).toEqual({
+      text: {
+        reranks: 1,
+        averageMs: 100,
+        considered: 4,
+        kept: 3,
+        keptRate: 75,
+      },
+      images: emptyStats,
+    });
+    model.durationMs = 300;
+    await request("/search/text?q=private-query&token=fixture-session");
+    model.durationMs = 20;
+    expect(
+      await request("/search/images?q=private-query&token=fixture-session"),
+    ).toEqual([imageResults[1], imageResults[2], imageResults[3]]);
+    expect((await request("/status")).reranker).toEqual({
+      reranks: 3,
+      averageMs: 140,
+      considered: 12,
+      kept: 9,
+      keptRate: 75,
+      fallbackApplied: 0,
+      skippedUnhealthy: 0,
+      failed: 0,
+      byType: {
+        text: {
+          reranks: 2,
+          averageMs: 200,
+          considered: 8,
+          kept: 6,
+          keptRate: 75,
+        },
+        images: {
+          reranks: 1,
+          averageMs: 20,
+          considered: 4,
+          kept: 3,
+          keptRate: 75,
+        },
+      },
+    });
+  });
+
+  it("keeps startup and empty searches at zero for both categories", async () => {
+    const before = (await request("/status")).reranker;
+    expect(before).toEqual({
+      ...emptyStats,
+      fallbackApplied: 0,
+      skippedUnhealthy: 0,
+      failed: 0,
+      byType: { text: emptyStats, images: emptyStats },
+    });
+    const { fetchSearXNG } = await import("./webSearchService.ts");
+    vi.mocked(fetchSearXNG).mockResolvedValue([]);
+    for (const type of ["text", "images"]) {
+      expect(
+        await request(`/search/${type}?q=empty&token=fixture-session`),
+      ).toEqual([]);
+    }
+    expect((await request("/status")).reranker).toEqual(before);
+  });
+
+  it("keeps unhealthy and failed counts at the top level without counting a successful rerank", async () => {
+    vi.spyOn(console, "warn").mockImplementation(() => {});
+    vi.spyOn(console, "error").mockImplementation(() => {});
+    for (const type of ["text", "images"]) {
+      model.healthy = false;
+      expect(
+        await request(`/search/${type}?q=unavailable&token=fixture-session`),
+      ).toEqual(type === "text" ? textResults : imageResults);
+      model.healthy = true;
+      model.fail = true;
+      expect(
+        await request(`/search/${type}?q=failed&token=fixture-session`),
+      ).toEqual(type === "text" ? textResults : imageResults);
+      model.fail = false;
+    }
+    expect((await request("/status")).reranker).toEqual({
+      ...emptyStats,
+      fallbackApplied: 0,
+      skippedUnhealthy: 2,
+      failed: 2,
+      byType: { text: emptyStats, images: emptyStats },
+    });
+  });
+});

server/searchEndpointServerHook.test.ts

@@ -71,8 +71,9 @@ describe("searchEndpointServerHook", () => {
       shouldContinue: true,
     });
     vi.mocked(getRerankerStatus).mockResolvedValue(false);
-    vi.mocked(rankSearchResults).mockImplementation(async (_query, results) =>
-      results.map(([title, content, url]) => [title, content, url, 0]),
+    vi.mocked(rankSearchResults).mockImplementation(
+      async (_query, _searchType, results) =>
+        results.map(([title, content, url]) => [title, content, url, 0]),
     );
   });
 
@@ -210,6 +211,7 @@ describe("searchEndpointServerHook", () => {
     expect(fetchSearXNG).toHaveBeenCalledWith("test", "text", 5);
     expect(rankSearchResults).toHaveBeenCalledWith(
       "test",
+      "text",
       [
         ["A", "snippet a", "https://a.com"],
         ["B", "snippet b", "https://b.com"],

server/searchEndpointServerHook.ts

@@ -11,6 +11,7 @@ import {
   incrementGraphicalSearchesSinceLastRestart,
   incrementTextualSearchesSinceLastRestart,
   recordSearchDuration,
+  type SearchType,
 } from "./searchesSinceLastRestart.ts";
 import { sendValidationError } from "./utils/httpResponse.ts";
 import { fetchSearXNG } from "./webSearchService.ts";
@@ -46,8 +47,9 @@ type ImageResult = [
 
 async function handleRanking(
   query: string,
+  searchType: SearchType,
   results: [title: string, content: string, url: string][],
-  isTextSearch?: boolean,
+  preserveTopResults = false,
 ): Promise<[title: string, content: string, url: string, score?: number][]> {
   const isRerankerHealthy = await getRerankerStatus();
   if (!isRerankerHealthy) {
@@ -57,7 +59,12 @@ async function handleRanking(
 
   try {
     if (isRerankerHealthy) {
-      return await rankSearchResults(query, results, isTextSearch);
+      return await rankSearchResults(
+        query,
+        searchType,
+        results,
+        preserveTopResults,
+      );
     }
     return results;
   } catch (error) {
@@ -128,7 +135,12 @@ export function searchEndpointServerHook<
 
       if (isTextSearch) {
         const results = searxngResults as TextResult[];
-        const rankedResults = await handleRanking(query, results, true);
+        const rankedResults = await handleRanking(
+          query,
+          searchType,
+          results,
+          true,
+        );
 
         incrementTextualSearchesSinceLastRestart();
 
@@ -139,7 +151,11 @@ export function searchEndpointServerHook<
         const resultsText = results.map(
           ([title, url]) => [title?.slice(0, 100) || "", "", url] as TextResult,
         );
-        const rankedResults = await handleRanking(query, resultsText);
+        const rankedResults = await handleRanking(
+          query,
+          searchType,
+          resultsText,
+        );
 
         const processedResults = rankedResults
           .map(([title, , rankedResultUrl]) => {

server/statusEndpointServerHook.test.ts

@@ -67,6 +67,17 @@ describe("statusEndpointServerHook", () => {
     });
   });
 
+  it("publishes the page-read circuit count beside the outcome counters", async () => {
+    const status = await callStatus();
+
+    // Counts and nothing else: the hosts the breaker boxes come from search
+    // results, so naming one here would say something about a query.
+    expect(status.pageReads).toMatchObject({
+      circuitOpens: expect.any(Number),
+      skipped: { skippedByBreaker: expect.any(Number) },
+    });
+  });
+
   it("keeps startedAt fixed at the restart while uptime advances", async () => {
     const first = await callStatus();
     expect(typeof first.startedAt).toBe("string");

server/statusEndpointServerHook.ts

@@ -3,6 +3,7 @@ import type { PreviewServer, ViteDevServer } from "vite";
 import { getAuthorizationStats } from "./authorizationSinceLastRestart.ts";
 import { getBiEncoderStatus } from "./biEncoderService.ts";
 import { getInferenceStats } from "./inferencesSinceLastRestart.ts";
+import { getPageReadCircuitStats } from "./pageReadHostBreaker.ts";
 import { getPageReadStats } from "./pageReadsSinceLastRestart.ts";
 import { getRerankerStatus } from "./rerankerService.ts";
 import { getRerankingStats } from "./rerankingSinceLastRestart.ts";
@@ -85,7 +86,10 @@ export function statusEndpointServerHook<
       biEncoderServiceStatus,
       rerankerServiceStatus,
       webSearchServiceStatus,
-      pageReads: getPageReadStats(),
+      pageReads: {
+        ...getPageReadStats(),
+        ...getPageReadCircuitStats(),
+      },
       authorization: getAuthorizationStats(),
       inference: getInferenceStats(),
       activeSessions: getActiveSessionsAmount(),

server/thumbnailEndpointServerHook.test.ts

@@ -18,6 +18,21 @@ vi.mock("./searchesSinceLastRestart", () => ({
   recordThumbnailBlocked: vi.fn(),
 }));
 
+// The hook's connect goes through the pinned request, so that is the seam
+// under test here; the capped reader stays real so the size-cap semantics
+// this file asserts are the shipped ones. The pin itself is covered by
+// pinnedFetch.test.ts.
+const pinnedRequestMock = vi.hoisted(() => vi.fn());
+
+vi.mock("./utils/pinnedFetch", async (importOriginal) => {
+  const actual =
+    await importOriginal<typeof import("./utils/pinnedFetch.ts")>();
+  return {
+    ...actual,
+    requestPinnedToVettedAddress: pinnedRequestMock,
+  };
+});
+
 import { handleTokenVerification } from "./handleTokenVerification";
 import {
   recordThumbnailBlocked,
@@ -30,9 +45,6 @@ import {
   thumbnailEndpointServerHook,
 } from "./thumbnailEndpointServerHook";
 
-const mockFetch = vi.fn();
-vi.stubGlobal("fetch", mockFetch);
-
 function createRequest(url: string): IncomingMessage {
   return {
     url,
@@ -75,20 +87,34 @@ function requestUrlFor(thumbnailUrl: string): string {
   return `/thumbnail?u=${encodeURIComponent(thumbnailUrl)}&token=abc`;
 }
 
+// Shaped like the IncomingMessage the pinned request resolves with: the
+// fields the hook reads, plus the async iteration readCappedStream drives.
+function fakeIncoming(
+  bytes: Uint8Array,
+  statusCode: number,
+  headers: Record<string, string>,
+): IncomingMessage {
+  return {
+    statusCode,
+    headers,
+    destroy: vi.fn(),
+    async *[Symbol.asyncIterator]() {
+      if (bytes.byteLength > 0) yield bytes;
+    },
+  } as unknown as IncomingMessage;
+}
+
 function imageResponse(
   bytes: Uint8Array<ArrayBuffer> = new Uint8Array([1, 2, 3]),
   contentType = "image/jpeg",
-): Response {
-  return new Response(bytes, {
-    status: 200,
-    headers: { "content-type": contentType },
-  });
+): IncomingMessage {
+  return fakeIncoming(bytes, 200, { "content-type": contentType });
 }
 
 describe("thumbnailEndpointServerHook", () => {
   beforeEach(() => {
     vi.clearAllMocks();
-    mockFetch.mockReset();
+    pinnedRequestMock.mockReset();
     lookupMock.mockResolvedValue([{ address: "93.184.216.34", family: 4 }]);
     vi.mocked(handleTokenVerification).mockResolvedValue({
       shouldContinue: true,
@@ -107,7 +133,7 @@ describe("thumbnailEndpointServerHook", () => {
     await handler(createRequest("/status"), createResponse(), next);
 
     expect(next).toHaveBeenCalled();
-    expect(mockFetch).not.toHaveBeenCalled();
+    expect(pinnedRequestMock).not.toHaveBeenCalled();
   });
 
   it("stops processing when token verification fails", async () => {
@@ -122,7 +148,7 @@ describe("thumbnailEndpointServerHook", () => {
       vi.fn(),
     );
 
-    expect(mockFetch).not.toHaveBeenCalled();
+    expect(pinnedRequestMock).not.toHaveBeenCalled();
     expect(recordThumbnailRequested).not.toHaveBeenCalled();
   });
 
@@ -136,12 +162,12 @@ describe("thumbnailEndpointServerHook", () => {
     expect(response.end).toHaveBeenCalledWith(
       JSON.stringify({ error: "Missing thumbnail URL" }),
     );
-    expect(mockFetch).not.toHaveBeenCalled();
+    expect(pinnedRequestMock).not.toHaveBeenCalled();
     expect(recordThumbnailRequested).not.toHaveBeenCalled();
   });
 
   it("serves a fetched image with its normalized content type and a private cache", async () => {
-    mockFetch.mockResolvedValue(
+    pinnedRequestMock.mockResolvedValue(
       imageResponse(new Uint8Array([1, 2, 3]), "IMAGE/JPEG; charset=binary"),
     );
     const handler = getRegisteredHandler();
@@ -179,7 +205,7 @@ describe("thumbnailEndpointServerHook", () => {
   });
 
   it("verifies through the thumbnail's own rate-limit budget", async () => {
-    mockFetch.mockResolvedValue(imageResponse());
+    pinnedRequestMock.mockResolvedValue(imageResponse());
     const { thumbnailRateLimiter } = await import("./verifyTokenAndRateLimit");
     const handler = getRegisteredHandler();
 
@@ -202,14 +228,14 @@ describe("thumbnailEndpointServerHook", () => {
 
   it("serves a repeat request from the cache without fetching again", async () => {
     const url = publicThumbnailUrl();
-    // A fresh Response per call: a body can only be read once.
-    mockFetch.mockImplementation(async () => imageResponse());
+    // A fresh response per call: a body can only be read once.
+    pinnedRequestMock.mockImplementation(async () => imageResponse());
     const handler = getRegisteredHandler();
 
     await handler(createRequest(requestUrlFor(url)), createResponse(), vi.fn());
     await handler(createRequest(requestUrlFor(url)), createResponse(), vi.fn());
 
-    expect(mockFetch).toHaveBeenCalledTimes(1);
+    expect(pinnedRequestMock).toHaveBeenCalledTimes(1);
     // Both requests are demand, so both count even though only one fetched.
     expect(recordThumbnailRequested).toHaveBeenCalledTimes(2);
   });
@@ -226,7 +252,7 @@ describe("thumbnailEndpointServerHook", () => {
     );
 
     expect(response.statusCode).toBe(403);
-    expect(mockFetch).not.toHaveBeenCalled();
+    expect(pinnedRequestMock).not.toHaveBeenCalled();
     expect(recordThumbnailBlocked).toHaveBeenCalledTimes(1);
     expect(recordThumbnailDropped).toHaveBeenCalledTimes(1);
   });
@@ -242,7 +268,7 @@ describe("thumbnailEndpointServerHook", () => {
     );
 
     expect(response.statusCode).toBe(403);
-    expect(mockFetch).not.toHaveBeenCalled();
+    expect(pinnedRequestMock).not.toHaveBeenCalled();
     expect(recordThumbnailBlocked).toHaveBeenCalledTimes(1);
     expect(recordThumbnailDropped).toHaveBeenCalledTimes(1);
   });
@@ -255,18 +281,43 @@ describe("thumbnailEndpointServerHook", () => {
     await handler(createRequest(requestUrlFor(longUrl)), response, vi.fn());
 
     expect(response.statusCode).toBe(400);
-    expect(mockFetch).not.toHaveBeenCalled();
+    expect(pinnedRequestMock).not.toHaveBeenCalled();
     expect(recordThumbnailRequested).not.toHaveBeenCalled();
   });
 
+  it("pins the address vetted at the first DNS answer, never the rebound one", async () => {
+    // The rebinding attack: the first answer is public, the second — the
+    // one a re-resolving fetch would get at connect time — is private.
+    // The pinned connect must use the vetted answer, and the private one
+    // must never be dialed.
+    lookupMock
+      .mockResolvedValueOnce([{ address: "93.184.216.34", family: 4 }])
+      .mockResolvedValue([{ address: "10.0.0.7", family: 4 }]);
+    pinnedRequestMock.mockResolvedValue(imageResponse());
+    const handler = getRegisteredHandler();
+    const response = createResponse();
+
+    await handler(
+      createRequest(requestUrlFor(publicThumbnailUrl())),
+      response,
+      vi.fn(),
+    );
+
+    expect(response.statusCode).toBe(200);
+    expect(pinnedRequestMock).toHaveBeenCalledTimes(1);
+    expect(pinnedRequestMock.mock.calls[0][1]).toBe("93.184.216.34");
+    expect(
+      pinnedRequestMock.mock.calls.some((call) => call[1] === "10.0.0.7"),
+    ).toBe(false);
+  });
+
   it("does not follow a redirect into a private address", async () => {
     lookupMock
       .mockResolvedValueOnce([{ address: "93.184.216.34", family: 4 }])
       .mockResolvedValue([{ address: "10.0.0.7", family: 4 }]);
-    mockFetch.mockResolvedValueOnce(
-      new Response(null, {
-        status: 302,
-        headers: { location: "http://10.0.0.7/thumb.jpg" },
+    pinnedRequestMock.mockResolvedValueOnce(
+      fakeIncoming(new Uint8Array(), 302, {
+        location: "http://10.0.0.7/thumb.jpg",
       }),
     );
     const handler = getRegisteredHandler();
@@ -279,12 +330,17 @@ describe("thumbnailEndpointServerHook", () => {
     );
 
     expect(response.statusCode).toBe(403);
-    expect(mockFetch).toHaveBeenCalledTimes(1);
+    // The redirect target was refused at validation: its socket was never
+    // opened, so the private address was never connected to.
+    expect(pinnedRequestMock).toHaveBeenCalledTimes(1);
+    expect(pinnedRequestMock.mock.calls[0][1]).toBe("93.184.216.34");
     expect(recordThumbnailBlocked).toHaveBeenCalledTimes(1);
   });
 
   it("answers 502 with no-store when the upstream returns an error status", async () => {
-    mockFetch.mockResolvedValue(new Response(null, { status: 404 }));
+    pinnedRequestMock.mockResolvedValue(
+      fakeIncoming(new Uint8Array(), 404, {}),
+    );
     const handler = getRegisteredHandler();
     const response = createResponse();
 
@@ -306,10 +362,9 @@ describe("thumbnailEndpointServerHook", () => {
   });
 
   it("answers 502 when the upstream answer is not an accepted image type", async () => {
-    mockFetch.mockResolvedValue(
-      new Response("<html></html>", {
-        status: 200,
-        headers: { "content-type": "text/html" },
+    pinnedRequestMock.mockResolvedValue(
+      fakeIncoming(new TextEncoder().encode("<html></html>"), 200, {
+        "content-type": "text/html",
       }),
     );
     const handler = getRegisteredHandler();
@@ -327,11 +382,12 @@ describe("thumbnailEndpointServerHook", () => {
 
   it("answers 502 when the upstream answer is an SVG", async () => {
     // An SVG is a document: served same-origin it would run its scripts.
-    mockFetch.mockResolvedValue(
-      new Response("<svg><script>alert(1)</script></svg>", {
-        status: 200,
-        headers: { "content-type": "image/svg+xml" },
-      }),
+    pinnedRequestMock.mockResolvedValue(
+      fakeIncoming(
+        new TextEncoder().encode("<svg><script>alert(1)</script></svg>"),
+        200,
+        { "content-type": "image/svg+xml" },
+      ),
     );
     const handler = getRegisteredHandler();
     const response = createResponse();
@@ -347,11 +403,8 @@ describe("thumbnailEndpointServerHook", () => {
   });
 
   it("does not follow a redirect with a malformed Location", async () => {
-    mockFetch.mockResolvedValueOnce(
-      new Response(null, {
-        status: 302,
-        headers: { location: "http://[" },
-      }),
+    pinnedRequestMock.mockResolvedValueOnce(
+      fakeIncoming(new Uint8Array(), 302, { location: "http://[" }),
     );
     const handler = getRegisteredHandler();
     const response = createResponse();
@@ -363,14 +416,14 @@ describe("thumbnailEndpointServerHook", () => {
     );
 
     expect(response.statusCode).toBe(502);
-    expect(mockFetch).toHaveBeenCalledTimes(1);
+    expect(pinnedRequestMock).toHaveBeenCalledTimes(1);
     expect(recordThumbnailDropped).toHaveBeenCalledTimes(1);
   });
 
   it("does not cache a failure, so the next load retries the upstream", async () => {
     const url = publicThumbnailUrl();
-    mockFetch
-      .mockResolvedValueOnce(new Response(null, { status: 404 }))
+    pinnedRequestMock
+      .mockResolvedValueOnce(fakeIncoming(new Uint8Array(), 404, {}))
       .mockResolvedValueOnce(imageResponse());
     const handler = getRegisteredHandler();
 
@@ -381,15 +434,12 @@ describe("thumbnailEndpointServerHook", () => {
     const second = createResponse();
     await handler(createRequest(requestUrlFor(url)), second, vi.fn());
     expect(second.statusCode).toBe(200);
-    expect(mockFetch).toHaveBeenCalledTimes(2);
+    expect(pinnedRequestMock).toHaveBeenCalledTimes(2);
   });
 
   it("answers 502 when the upstream body is empty", async () => {
-    mockFetch.mockResolvedValue(
-      new Response(new Uint8Array(), {
-        status: 200,
-        headers: { "content-type": "image/jpeg" },
-      }),
+    pinnedRequestMock.mockResolvedValue(
+      fakeIncoming(new Uint8Array(), 200, { "content-type": "image/jpeg" }),
     );
     const handler = getRegisteredHandler();
     const response = createResponse();
@@ -422,7 +472,7 @@ describe("thumbnailEndpointServerHook", () => {
       await handled;
 
       expect(response.statusCode).toBe(502);
-      expect(mockFetch).not.toHaveBeenCalled();
+      expect(pinnedRequestMock).not.toHaveBeenCalled();
       expect(recordThumbnailDropped).toHaveBeenCalledTimes(1);
     } finally {
       vi.useRealTimers();
@@ -434,10 +484,10 @@ describe("thumbnailEndpointServerHook", () => {
     const thumbnailTimeoutMs = 3000;
     vi.useFakeTimers();
     try {
-      mockFetch.mockImplementation(
-        (_url: string, init?: RequestInit) =>
+      pinnedRequestMock.mockImplementation(
+        (_url: URL, _address: string, options?: { signal?: AbortSignal }) =>
           new Promise((_resolve, reject) => {
-            init?.signal?.addEventListener("abort", () =>
+            options?.signal?.addEventListener("abort", () =>
               reject(new Error("The operation was aborted")),
             );
           }),
@@ -460,10 +510,9 @@ describe("thumbnailEndpointServerHook", () => {
   });
 
   it("answers 502 after the redirect budget is spent", async () => {
-    mockFetch.mockResolvedValue(
-      new Response(null, {
-        status: 302,
-        headers: { location: "https://thumbs.example.com/loop.jpg" },
+    pinnedRequestMock.mockResolvedValue(
+      fakeIncoming(new Uint8Array(), 302, {
+        location: "https://thumbs.example.com/loop.jpg",
       }),
     );
     const handler = getRegisteredHandler();
@@ -477,13 +526,13 @@ describe("thumbnailEndpointServerHook", () => {
 
     expect(response.statusCode).toBe(502);
     // One request per hop, four hops for three redirects.
-    expect(mockFetch).toHaveBeenCalledTimes(4);
+    expect(pinnedRequestMock).toHaveBeenCalledTimes(4);
     expect(recordThumbnailDropped).toHaveBeenCalledTimes(1);
   });
 
   it("truncates an oversized body to the byte cap", async () => {
     const bigBody = new Uint8Array(600_000).fill(7);
-    mockFetch.mockResolvedValue(imageResponse(bigBody));
+    pinnedRequestMock.mockResolvedValue(imageResponse(bigBody));
     const handler = getRegisteredHandler();
     const response = createResponse();
 
@@ -500,8 +549,8 @@ describe("thumbnailEndpointServerHook", () => {
   });
 
   it("evicts the least recently used entry, not the least recently loaded one", async () => {
-    // A fresh Response per call: a body can only be read once.
-    mockFetch.mockImplementation(async () => imageResponse());
+    // A fresh response per call: a body can only be read once.
+    pinnedRequestMock.mockImplementation(async () => imageResponse());
     const handler = getRegisteredHandler();
     // A clean slate, so the eviction side is exactly where this test puts it.
     resetThumbnailCache();
@@ -511,7 +560,8 @@ describe("thumbnailEndpointServerHook", () => {
     const load = (url: string) =>
       handler(createRequest(requestUrlFor(url)), createResponse(), vi.fn());
     const fetched = (url: string) =>
-      mockFetch.mock.calls.filter((call) => String(call[0]) === url).length;
+      pinnedRequestMock.mock.calls.filter((call) => String(call[0]) === url)
+        .length;
 
     await load(firstUrl);
     await load(secondUrl);

server/thumbnailEndpointServerHook.ts

@@ -1,4 +1,4 @@
-import type { ServerResponse } from "node:http";
+import type { IncomingMessage, ServerResponse } from "node:http";
 import type { PreviewServer, ViteDevServer } from "vite";
 import { handleTokenVerification } from "./handleTokenVerification.ts";
 import {
@@ -6,10 +6,57 @@ import {
   recordThumbnailDropped,
   recordThumbnailRequested,
 } from "./searchesSinceLastRestart.ts";
-import { resolvePublicUrl } from "./utils/publicUrl.ts";
-import { readCappedBytes, safeEndResponse } from "./utils/streamUtils.ts";
+import {
+  readCappedStream,
+  requestPinnedToVettedAddress,
+} from "./utils/pinnedFetch.ts";
+import { resolvePublicUrlAndAddress } from "./utils/publicUrl.ts";
+import { safeEndResponse } from "./utils/streamUtils.ts";
 import { thumbnailRateLimiter } from "./verifyTokenAndRateLimit.ts";
 
+/**
+ * Security review for issue #2524 (audited 2026-09-17). What each item of
+ * the audit checklist maps to:
+ *
+ * - Private ranges (10/8, 172.16/12, 192.168/16), link-local including
+ *   the cloud metadata address 169.254.169.254, loopback (127/8, ::1),
+ *   CGNAT 100.64/10, multicast and reserved space, and the IPv4-mapped /
+ *   IPv4-compatible IPv6 spellings: `BLOCKED_CIDRS` and `isBlockedAddress`
+ *   in `server/utils/publicUrl.ts`, enforced per hop by
+ *   `resolvePublicUrlAndAddress`. NAT64/6to4/Teredo are refused wholesale
+ *   because an embedded IPv4 cannot be vetted.
+ * - DNS rebinding: closed. The vetted address is pinned to the socket by
+ *   `requestPinnedToVettedAddress` (`server/utils/pinnedFetch.ts`); the
+ *   connect never re-resolves, so a second, private DNS answer cannot be
+ *   reached. TLS SNI and certificate verification still run against the
+ *   original hostname, as does the `Host` header.
+ * - URL length: `MAX_THUMBNAIL_URL_LENGTH` (2048), refused before any
+ *   work is done.
+ * - Schemes: http/https only, in `resolvePublicUrlAndAddress`.
+ * - Redirects: followed by hand below, every hop re-validated under one
+ *   shared deadline.
+ * - Content-Type: raster-only allowlist below; SVG is deliberately
+ *   excluded because served same-origin it is a script-carrying document.
+ * - Response size: `MAX_THUMBNAIL_BYTES` via `readCappedStream`.
+ * - Timeout: `THUMBNAIL_TIMEOUT_MS`, shared across DNS and every hop.
+ * - Rate limiting: per client IP via `thumbnailRateLimiter`, a dedicated
+ *   60-per-10s budget so a grid of tiles cannot exhaust the search
+ *   budget. There is no separate global limiter: the per-client limiter
+ *   plus the shared token gate bound the total, and a global cap would let
+ *   one caller's grid starve every other user's tiles.
+ *
+ * Residual risk after this change: the endpoint still fetches arbitrary
+ * public URLs, so a caller can use it to GET any public host — bounded by
+ * the rate limiter, the size cap and the timeout, and useful only for
+ * raster images. The pinned address is the one DNS gave at vetting time; a
+ * host whose DNS later repoints stays unreachable here until its records
+ * are re-checked, which is the pin working, not failing. Content itself is
+ * untrusted: served same-origin, it carries `nosniff` and a sandbox CSP,
+ * and only raster types pass the allowlist, so no script-carrying format
+ * reaches a browser. `/page-content` has not adopted the pin yet and still
+ * carries the old rebinding residual (see `resolvePublicUrl`'s docstring).
+ */
+
 /**
  * Off the critical path: the search response no longer waits on a thumbnail,
  * so the budget is set against how long a user tolerates a placeholder tile,
@@ -109,9 +156,10 @@ function isRedirect(status: number): boolean {
 }
 
 /**
- * `resolvePublicUrl`'s DNS lookup does not take a signal, so it is raced
- * against the hop deadline: a hanging resolver would otherwise add its own
- * timeout on top of the one that is documented to bound the whole chain.
+ * `resolvePublicUrlAndAddress`'s DNS lookup does not take a signal, so it
+ * is raced against the hop deadline: a hanging resolver would otherwise
+ * add its own timeout on top of the one that is documented to bound the
+ * whole chain.
  */
 function timeoutError(): Error {
   const error = new Error("The operation was aborted");
@@ -119,18 +167,18 @@ function timeoutError(): Error {
   return error;
 }
 
-function resolveWithinDeadline(
+function resolveWithinDeadline<T>(
   deadline: AbortSignal,
-  work: Promise<URL>,
-): Promise<URL> {
+  work: Promise<T>,
+): Promise<T> {
   if (deadline.aborted) return Promise.reject(timeoutError());
-  return new Promise<URL>((resolve, reject) => {
+  return new Promise<T>((resolve, reject) => {
     const onAbort = () => reject(timeoutError());
     deadline.addEventListener("abort", onAbort, { once: true });
     work.then(
-      (url) => {
+      (value) => {
         deadline.removeEventListener("abort", onAbort);
-        resolve(url);
+        resolve(value);
       },
       (error) => {
         deadline.removeEventListener("abort", onAbort);
@@ -141,10 +189,14 @@ function resolveWithinDeadline(
 }
 
 /**
- * Follows redirects by hand so that every hop is validated, the way
- * `pageContentService` does: `redirect: "follow"` would let a public host
- * bounce the server into a private address. The chain shares one deadline, so
- * a redirector cannot buy extra time per hop.
+ * Follows redirects by hand so that every hop is validated and pinned, the
+ * way `pageContentService` follows them: `redirect: "follow"` would let a
+ * public host bounce the server into a private address. The chain shares one
+ * deadline, so a redirector cannot buy extra time per hop.
+ *
+ * Each hop connects to the address `resolvePublicUrlAndAddress` vetted, via
+ * `requestPinnedToVettedAddress`, so the address checked is the address
+ * reached: a second, private DNS answer has no path to the socket.
  *
  * The URL is client-supplied (it arrives as a query parameter), so on failure
  * it is not logged, matching `/page-content`; the outcome is counted instead.
@@ -155,8 +207,12 @@ async function fetchThumbnail(rawUrl: string): Promise<ThumbnailOutcome> {
 
   for (let hop = 0; hop <= MAX_THUMBNAIL_REDIRECTS; hop++) {
     let url: URL;
+    let address: string;
     try {
-      url = await resolveWithinDeadline(deadline, resolvePublicUrl(target));
+      ({ url, address } = await resolveWithinDeadline(
+        deadline,
+        resolvePublicUrlAndAddress(target),
+      ));
     } catch (error) {
       if (error instanceof Error && error.name === "TimeoutError") {
         return { kind: "failed", reason: "TimeoutError" };
@@ -164,10 +220,9 @@ async function fetchThumbnail(rawUrl: string): Promise<ThumbnailOutcome> {
       return { kind: "blocked" };
     }
 
-    let response: Response;
+    let response: IncomingMessage;
     try {
-      response = await fetch(url, {
-        redirect: "manual",
+      response = await requestPinnedToVettedAddress(url, address, {
         signal: deadline,
       });
     } catch (error) {
@@ -177,9 +232,10 @@ async function fetchThumbnail(rawUrl: string): Promise<ThumbnailOutcome> {
       };
     }
 
-    const location = response.headers.get("location");
-    if (isRedirect(response.status) && location) {
-      await response.body?.cancel().catch(() => {});
+    const location = response.headers.location;
+    if (isRedirect(response.statusCode ?? 0) && location) {
+      // Tear the socket down rather than drain a body nobody asked for.
+      response.destroy();
       try {
         target = new URL(location, url).toString();
       } catch {
@@ -188,17 +244,18 @@ async function fetchThumbnail(rawUrl: string): Promise<ThumbnailOutcome> {
       continue;
     }
 
-    if (!response.ok) {
-      await response.body?.cancel().catch(() => {});
-      return { kind: "failed", reason: `HTTP ${response.status}` };
+    const status = response.statusCode ?? 0;
+    if (status < 200 || status >= 300) {
+      response.destroy();
+      return { kind: "failed", reason: `HTTP ${status}` };
     }
 
-    const normalizedType = (response.headers.get("content-type") ?? "")
+    const normalizedType = (response.headers["content-type"] ?? "")
       .split(";")[0]
       .trim()
       .toLowerCase();
     if (!ACCEPTED_CONTENT_TYPES.includes(normalizedType)) {
-      await response.body?.cancel().catch(() => {});
+      response.destroy();
       return {
         kind: "failed",
         reason: normalizedType
@@ -209,7 +266,7 @@ async function fetchThumbnail(rawUrl: string): Promise<ThumbnailOutcome> {
 
     let bytes: Uint8Array;
     try {
-      ({ bytes } = await readCappedBytes(response, MAX_THUMBNAIL_BYTES));
+      ({ bytes } = await readCappedStream(response, MAX_THUMBNAIL_BYTES));
     } catch (error) {
       return {
         kind: "failed",
@@ -309,8 +366,8 @@ export function thumbnailEndpointServerHook<
         // `dropped` is the total of what never reached the client, blocked
         // included, so `requested` minus `dropped` stays the served count.
         recordThumbnailDropped();
-        // Covers both refusals resolvePublicUrl reports the same way: a host
-        // in private space and a host that does not resolve.
+        // Covers both refusals resolvePublicUrlAndAddress reports the same
+        // way: a host in private space and a host that does not resolve.
         serveError(
           response,
           403,

server/utils/circuitBreaker.test.ts

@@ -30,7 +30,66 @@ describe("CircuitBreaker", () => {
         }),
       ).rejects.toThrow("function error");
     });
+
+    it("resets failure count on success so spaced failures do not open the circuit", async () => {
+      const cb = new CircuitBreaker();
+      const fail = () =>
+        expect(
+          cb.execute("key", async () => {
+            throw new Error("transient");
+          }),
+        ).rejects.toThrow("transient");
+
+      for (let attempt = 0; attempt < 4; attempt++) {
+        await fail();
+      }
+      await cb.execute("key", async () => "success");
+      for (let attempt = 0; attempt < 4; attempt++) {
+        await fail();
+      }
+
+      expect(cb.getState("key")).toBe("CLOSED");
+      expect(cb.getOpens("key")).toBe(0);
+    });
+  });
+  describe("allows, recordSuccess and recordFailure", () => {
+    afterEach(() => {
+      vi.useRealTimers();
+    });
+
+    it("drive the circuit through the same states execute does", () => {
+      vi.useFakeTimers();
+      const resetTimeout = 1000;
+      const cb = new CircuitBreaker({
+        failureThreshold: 2,
+        resetTimeout,
+        successThreshold: 1,
+      });
+
+      expect(cb.allows("host")).toBe(true);
+      // A success under a circuit nothing has failed on has nothing to record.
+      cb.recordSuccess("host");
+      expect(cb.getState("host")).toBe("CLOSED");
+
+      cb.recordFailure("host");
+      cb.recordFailure("host");
+      expect(cb.getState("host")).toBe("OPEN");
+      expect(cb.allows("host")).toBe(false);
+      expect(cb.getOpens("host")).toBe(1);
+
+      // The clock moves without the reset timer firing, so the circuit is still
+      // open when asked: the caller that asks after the window is the probe.
+      vi.setSystemTime(Date.now() + resetTimeout + 1);
+      expect(cb.getState("host")).toBe("OPEN");
+      expect(cb.allows("host")).toBe(true);
+      expect(cb.getState("host")).toBe("HALF_OPEN");
+
+      cb.recordSuccess("host");
+      expect(cb.getState("host")).toBe("CLOSED");
+      expect(cb.getOpens("host")).toBe(1);
+    });
   });
+
   describe("getOpens", () => {
     afterEach(() => {
       vi.useRealTimers();

server/utils/circuitBreaker.ts

@@ -1,6 +1,7 @@
 type CircuitState = "CLOSED" | "OPEN" | "HALF_OPEN";
 
-interface CircuitBreakerOptions {
+/** Thresholds for one breaker, shared by every circuit it keeps. */
+export interface CircuitBreakerOptions {
   failureThreshold: number;
   resetTimeout: number;
   successThreshold: number;
@@ -31,26 +32,76 @@ export class CircuitBreaker {
 
   /** Runs `fn` under the named circuit, tripping it after repeated failures. */
   async execute<T>(key: string, fn: () => Promise<T>): Promise<T> {
-    const metrics = this.getOrCreateMetrics(key);
-
-    if (metrics.state === "OPEN") {
-      if (this.shouldAttemptReset(metrics)) {
-        metrics.state = "HALF_OPEN";
-      } else {
-        throw new Error(`Circuit breaker is open for ${key}`);
-      }
+    if (!this.allows(key)) {
+      throw new Error(`Circuit breaker is open for ${key}`);
     }
 
     try {
       const result = await fn();
-      this.recordSuccess(metrics);
+      this.recordSuccess(key);
       return result;
     } catch (error) {
-      this.recordFailure(metrics);
+      this.recordFailure(key);
       throw error;
     }
   }
 
+  /**
+   * Whether a call under the named circuit may go ahead right now. An open
+   * circuit whose reset timeout has elapsed becomes half-open here, so the
+   * caller that asked is the one that probes it. With `recordSuccess` and
+   * `recordFailure` this is `execute` taken apart, for a caller whose
+   * failures are return values rather than exceptions.
+   */
+  allows(key: string): boolean {
+    const metrics = this.metrics.get(key);
+    if (metrics?.state !== "OPEN") return true;
+
+    if (!this.shouldAttemptReset(metrics)) return false;
+
+    metrics.state = "HALF_OPEN";
+    return true;
+  }
+
+  /** A success under the named circuit: clears its failures, or closes it after a probe. */
+  recordSuccess(key: string): void {
+    // A circuit nothing has failed under is closed with no failures already,
+    // so a success on it has nothing to record and no entry to create.
+    const metrics = this.metrics.get(key);
+    if (!metrics) return;
+
+    if (metrics.state === "HALF_OPEN") {
+      metrics.successes++;
+      if (metrics.successes >= this.options.successThreshold) {
+        this.resetMetrics(metrics);
+      }
+    } else {
+      metrics.failures = 0;
+    }
+  }
+
+  /** A failure under the named circuit, opening it at the threshold. */
+  recordFailure(key: string): void {
+    const metrics = this.getOrCreateMetrics(key);
+
+    metrics.failures++;
+    metrics.lastFailure = Date.now();
+    metrics.successes = 0;
+
+    if (
+      metrics.failures >= this.options.failureThreshold &&
+      metrics.state !== "OPEN"
+    ) {
+      metrics.state = "OPEN";
+      metrics.opens++;
+      setTimeout(() => {
+        if (metrics.state === "OPEN") {
+          metrics.state = "HALF_OPEN";
+        }
+      }, this.options.resetTimeout);
+    }
+  }
+
   getState(key: string): CircuitState {
     return this.metrics.get(key)?.state || "CLOSED";
   }
@@ -89,36 +140,6 @@ export class CircuitBreaker {
     return now - metrics.lastFailure > this.options.resetTimeout;
   }
 
-  private recordSuccess(metrics: CircuitMetrics): void {
-    if (metrics.state === "HALF_OPEN") {
-      metrics.successes++;
-      if (metrics.successes >= this.options.successThreshold) {
-        this.resetMetrics(metrics);
-      }
-    } else {
-      metrics.failures = 0;
-    }
-  }
-
-  private recordFailure(metrics: CircuitMetrics): void {
-    metrics.failures++;
-    metrics.lastFailure = Date.now();
-    metrics.successes = 0;
-
-    if (
-      metrics.failures >= this.options.failureThreshold &&
-      metrics.state !== "OPEN"
-    ) {
-      metrics.state = "OPEN";
-      metrics.opens++;
-      setTimeout(() => {
-        if (metrics.state === "OPEN") {
-          metrics.state = "HALF_OPEN";
-        }
-      }, this.options.resetTimeout);
-    }
-  }
-
   private resetMetrics(metrics: CircuitMetrics): void {
     metrics.failures = 0;
     metrics.successes = 0;

server/utils/pinnedFetch.test.ts

@@ -0,0 +1,247 @@
+import http from "node:http";
+import type { RequestOptions } from "node:https";
+import type { AddressInfo } from "node:net";
+import { describe, expect, it, vi } from "vitest";
+
+// https is stubbed so the TLS options the pin relies on — servername, Host,
+// the lookup override — can be inspected without a certificate. http stays
+// real: one test below drives an actual socket through the pin.
+const httpsRequestMock = vi.hoisted(() => vi.fn());
+
+vi.mock("node:https", () => ({
+  default: { request: httpsRequestMock },
+}));
+
+import {
+  readCappedStream,
+  requestPinnedToVettedAddress,
+} from "./pinnedFetch.ts";
+
+type LookupFn = (
+  hostname: string,
+  options: unknown,
+  callback: (
+    error: NodeJS.ErrnoException | null,
+    addresses: string | { address: string; family: number }[],
+    family?: number,
+  ) => void,
+) => void;
+
+/**
+ * Runs a pinned https request against a stub that answers with an empty
+ * 200, and hands back the options the request was built with.
+ */
+async function captureHttpsOptions(
+  url: URL,
+  address: string,
+): Promise<RequestOptions> {
+  httpsRequestMock.mockImplementation(
+    (_options: RequestOptions, callback: (response: unknown) => void) => ({
+      on: () => undefined,
+      end: () => callback({ statusCode: 200, headers: {} }),
+    }),
+  );
+  await requestPinnedToVettedAddress(url, address);
+  const calls = httpsRequestMock.mock.calls;
+  return calls[calls.length - 1][0] as RequestOptions;
+}
+
+/** Drives the lookup override and reports what it answers with. */
+function resolveViaPin(
+  options: RequestOptions,
+  queriedHostname: string,
+  lookupOptions: unknown = {},
+): Promise<{ address: string; family: number }> {
+  const lookup = options.lookup as LookupFn;
+  return new Promise((resolve, reject) => {
+    lookup(queriedHostname, lookupOptions, (error, addresses, family) => {
+      if (error) reject(error);
+      else if (typeof addresses === "string")
+        resolve({ address: addresses, family: family ?? 0 });
+      else resolve(addresses[0]);
+    });
+  });
+}
+
+function hostOf(options: RequestOptions): string {
+  return (options.headers as Record<string, string>).Host;
+}
+
+describe("requestPinnedToVettedAddress", () => {
+  it("keeps the original hostname for TLS SNI and the Host header while dialling the vetted address", async () => {
+    const options = await captureHttpsOptions(
+      new URL("https://thumbs.example.com/a.jpg"),
+      "93.184.216.34",
+    );
+
+    // SNI and certificate verification follow the hostname, not the IP,
+    // so a wrong-host certificate still fails the request.
+    expect(options.hostname).toBe("thumbs.example.com");
+    expect(options.servername).toBe("thumbs.example.com");
+    expect(options.port).toBe(443);
+    expect(hostOf(options)).toBe("thumbs.example.com");
+  });
+
+  it("answers the lookup override with the pinned address without touching DNS", async () => {
+    const options = await captureHttpsOptions(
+      new URL("https://thumbs.example.com/a.jpg"),
+      "93.184.216.34",
+    );
+
+    // Whatever hostname the connect layer asks about, the answer is the
+    // vetted address: no resolver is consulted, so a second DNS answer
+    // cannot exist to be rebound.
+    const first = await resolveViaPin(options, "thumbs.example.com");
+    const second = await resolveViaPin(options, "anything-else.test");
+    expect(first).toEqual({ address: "93.184.216.34", family: 4 });
+    expect(second).toEqual({ address: "93.184.216.34", family: 4 });
+
+    // The Happy Eyeballs shape: asked for every answer, the pin hands
+    // back exactly one — the vetted address.
+    const all = await resolveViaPin(options, "anything.test", { all: true });
+    expect(all).toEqual({ address: "93.184.216.34", family: 4 });
+  });
+
+  it("brackets an IPv6 host in the Host header and reports family 6", async () => {
+    const address = "2001:db8:85a3::8a2e:370:7334";
+    const options = await captureHttpsOptions(
+      new URL(`https://[${address}]/x.png`),
+      address,
+    );
+
+    expect(options.hostname).toBe(address);
+    expect(hostOf(options)).toBe(`[${address}]`);
+    const resolved = await resolveViaPin(options, "anything.test");
+    expect(resolved).toEqual({ address, family: 6 });
+  });
+
+  it("preserves a non-default port through the pinned connect and the Host header", async () => {
+    const options = await captureHttpsOptions(
+      new URL("https://thumbs.example.com:8443/a.jpg"),
+      "93.184.216.34",
+    );
+
+    expect(options.port).toBe(8443);
+    expect(hostOf(options)).toBe("thumbs.example.com:8443");
+  });
+
+  it("omits the default port from the Host header", async () => {
+    const options = await captureHttpsOptions(
+      new URL("https://thumbs.example.com:443/a.jpg"),
+      "93.184.216.34",
+    );
+
+    expect(options.port).toBe(443);
+    expect(hostOf(options)).toBe("thumbs.example.com");
+  });
+
+  it("refuses a scheme that is not http(s)", async () => {
+    await expect(
+      requestPinnedToVettedAddress(
+        new URL("ftp://example.com/x"),
+        "93.184.216.34",
+      ),
+    ).rejects.toThrow("Unsupported URL scheme");
+    expect(httpsRequestMock).not.toHaveBeenCalled();
+  });
+
+  it("dials the pinned address end-to-end while the server sees the original Host", async () => {
+    // pinned-host.test has no DNS record: the request can only succeed if
+    // the socket goes where the pin says, not where a resolver would.
+    let seenHost = "";
+    let seenTarget = "";
+    const server = http.createServer((request, response) => {
+      seenHost = request.headers.host ?? "";
+      seenTarget = request.url ?? "";
+      response.writeHead(200, { "content-type": "image/png" });
+      response.end(Buffer.from([9, 8, 7]));
+    });
+    await new Promise<void>((resolve) =>
+      server.listen(0, "127.0.0.1", resolve),
+    );
+    const { port } = server.address() as AddressInfo;
+
+    try {
+      const url = new URL(`http://pinned-host.test:${port}/img.png?v=1`);
+      const response = await requestPinnedToVettedAddress(url, "127.0.0.1");
+      const { bytes } = await readCappedStream(response, 1_000);
+
+      expect(response.statusCode).toBe(200);
+      expect(Array.from(bytes)).toEqual([9, 8, 7]);
+      expect(seenHost).toBe(`pinned-host.test:${port}`);
+      expect(seenTarget).toBe("/img.png?v=1");
+    } finally {
+      server.close();
+    }
+  });
+
+  it("refuses to run a request whose signal is already aborted", async () => {
+    const controller = new AbortController();
+    controller.abort();
+
+    await expect(
+      requestPinnedToVettedAddress(
+        new URL("http://example.com/x"),
+        "93.184.216.34",
+        { signal: controller.signal },
+      ),
+    ).rejects.toThrow();
+  });
+});
+
+describe("readCappedStream", () => {
+  function fakeStream(chunks: Uint8Array[]) {
+    return {
+      async *[Symbol.asyncIterator]() {
+        for (const chunk of chunks) yield chunk;
+      },
+      destroy: vi.fn(),
+    };
+  }
+
+  it("truncates at the cap and reports it", async () => {
+    const chunks = [new Uint8Array(300).fill(1), new Uint8Array(300).fill(2)];
+    const { bytes, truncated } = await readCappedStream(
+      fakeStream(chunks),
+      500,
+    );
+
+    expect(bytes.byteLength).toBe(500);
+    expect(truncated).toBe(true);
+    // The kept tail is the head of the second chunk, not the whole thing.
+    expect(Array.from(bytes.subarray(300))).toEqual(
+      new Array(200).fill(2) as number[],
+    );
+  });
+
+  it("reads an exact-fit body without truncation", async () => {
+    const { bytes, truncated } = await readCappedStream(
+      fakeStream([new Uint8Array([1, 2, 3, 4])]),
+      4,
+    );
+
+    expect(Array.from(bytes)).toEqual([1, 2, 3, 4]);
+    expect(truncated).toBe(false);
+  });
+
+  it("returns an empty buffer for an empty stream", async () => {
+    const { bytes, truncated } = await readCappedStream(fakeStream([]), 10);
+
+    expect(bytes.byteLength).toBe(0);
+    expect(truncated).toBe(false);
+  });
+
+  it("concatenates chunks below the cap in order", async () => {
+    const { bytes, truncated } = await readCappedStream(
+      fakeStream([
+        new Uint8Array([1]),
+        new Uint8Array([2, 3]),
+        new Uint8Array([4, 5]),
+      ]),
+      10,
+    );
+
+    expect(Array.from(bytes)).toEqual([1, 2, 3, 4, 5]);
+    expect(truncated).toBe(false);
+  });
+});

server/utils/pinnedFetch.ts

@@ -0,0 +1,126 @@
+import type { IncomingMessage } from "node:http";
+import http from "node:http";
+import https from "node:https";
+import { isIP } from "node:net";
+
+/**
+ * What the capped reader needs from a response body: an async stream of
+ * chunks that can be torn down early. `IncomingMessage` satisfies it, and so
+ * does a test double, which keeps the reader testable without a socket.
+ */
+export interface CappedReadable extends AsyncIterable<Uint8Array> {
+  destroy(): void;
+}
+
+/**
+ * GETs `url` over a socket pinned to `address`, the address the caller
+ * already vetted with `resolvePublicUrlAndAddress`. The custom `lookup`
+ * never touches DNS, so nothing re-resolves between the check and the
+ * connect: the rebinding window a check-then-`fetch` leaves open, where a
+ * second DNS answer could steer the socket into private space, is closed.
+ *
+ * The original hostname survives the pin three ways: it stays the request's
+ * `hostname`, so TLS SNI and certificate verification run against it, not
+ * against the IP being dialled, and the `Host` header is set explicitly so
+ * virtual-host routing is unaffected.
+ *
+ * @param url - The URL to fetch; its protocol picks http or https
+ * @param address - The vetted IP literal to connect to, IPv4 or IPv6
+ * @throws When the protocol is not http(s) or the request fails
+ */
+export function requestPinnedToVettedAddress(
+  url: URL,
+  address: string,
+  options: { signal?: AbortSignal } = {},
+): Promise<IncomingMessage> {
+  if (url.protocol !== "https:" && url.protocol !== "http:") {
+    return Promise.reject(new Error(`Unsupported URL scheme: ${url.protocol}`));
+  }
+
+  const isTls = url.protocol === "https:";
+  const defaultPort = isTls ? 443 : 80;
+  const hostname = url.hostname.replace(/^\[|\]$/g, "");
+  const port = url.port ? Number(url.port) : defaultPort;
+  // An IPv6 host needs its brackets back in the Host header.
+  const bracketed = isIP(hostname) === 6 ? `[${hostname}]` : hostname;
+  const hostHeader = port === defaultPort ? bracketed : `${bracketed}:${port}`;
+
+  const requestOptions: https.RequestOptions = {
+    path: `${url.pathname}${url.search}`,
+    method: "GET",
+    hostname,
+    port,
+    // SNI and certificate verification follow the hostname, not the pinned
+    // address, so a wrong-host certificate still fails the request.
+    servername: isTls ? hostname : undefined,
+    headers: { Host: hostHeader },
+    // The pin itself: the socket connects to the vetted address and no
+    // resolver is consulted, so the address checked is the address reached.
+    // Node's Happy Eyeballs path (autoSelectFamily, on by default since
+    // Node 20) asks the resolver with `all: true` and reads an array back,
+    // so the pin answers in whichever shape the connect layer asked for —
+    // one entry either way, the vetted address.
+    lookup: (
+      _hostname: string,
+      lookupOptions: unknown,
+      callback: (
+        error: NodeJS.ErrnoException | null,
+        addresses: string | { address: string; family: number }[],
+        family?: number,
+      ) => void,
+    ) => {
+      const family = isIP(address);
+      if ((lookupOptions as { all?: boolean } | undefined)?.all) {
+        callback(null, [{ address, family }]);
+      } else {
+        callback(null, address, family);
+      }
+    },
+    signal: options.signal,
+  };
+
+  return new Promise<IncomingMessage>((resolve, reject) => {
+    const request = isTls
+      ? https.request(requestOptions, resolve)
+      : http.request(requestOptions, resolve);
+    request.on("error", reject);
+    request.end();
+  });
+}
+
+/**
+ * Reads at most `maxBytes` from a Node response stream, the counterpart of
+ * `readCappedBytes` for the `node:http` world: a hostile or merely huge
+ * upstream must not be able to pin the server's memory, and a caller that
+ * only needs the head of the body gains nothing from the tail. Breaking out
+ * of the iteration destroys the stream, releasing the socket instead of
+ * draining the rest.
+ */
+export async function readCappedStream(
+  stream: CappedReadable,
+  maxBytes: number,
+): Promise<{ bytes: Uint8Array; truncated: boolean }> {
+  const chunks: Uint8Array[] = [];
+  let bytesRead = 0;
+  let truncated = false;
+
+  for await (const chunk of stream) {
+    const remaining = maxBytes - bytesRead;
+    if (chunk.byteLength > remaining) {
+      chunks.push(chunk.subarray(0, remaining));
+      bytesRead = maxBytes;
+      truncated = true;
+      break;
+    }
+    chunks.push(chunk);
+    bytesRead += chunk.byteLength;
+  }
+
+  const bytes = new Uint8Array(bytesRead);
+  let offset = 0;
+  for (const chunk of chunks) {
+    bytes.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return { bytes, truncated };
+}

server/utils/publicUrl.ts

@@ -124,22 +124,22 @@ export function isBlockedAddress(address: string): boolean {
 }
 
 /**
- * Validates a URL that the server is about to fetch on a client's behalf,
- * rejecting non-HTTP schemes and hosts that resolve into a blocked range.
+ * Validates a URL the server is about to fetch on a client's behalf and
+ * returns the address the fetch must connect to.
  *
- * The DNS answer is checked, not pinned: a name that resolves to a public
- * address here and to a private one microseconds later during the fetch would
- * still get through, and returning the extracted text is what would carry the
- * result back, so this is a real gap rather than one the response shape closes.
- * Pinning means connecting to the vetted IP with the hostname preserved for TLS,
- * which needs a custom `undici` dispatcher this project has no other use for.
- * The residual risk is up to one GET per redirect hop from inside the instance's
- * network, and the readable text of whatever those GETs return.
+ * Returning the vetted address is what makes pinning possible. A caller
+ * that validates here and then dials the hostname again (a plain `fetch`)
+ * re-resolves at connect time, and that second answer can differ from the
+ * one that was checked — the DNS-rebinding window. Handing the checked
+ * answer to `requestPinnedToVettedAddress` in `pinnedFetch.ts` makes the
+ * checked address and the connected address one and the same.
  *
- * @returns The parsed URL when it is safe to fetch
+ * @returns The parsed URL and the vetted IP literal to pin the connection to
  * @throws When the scheme is unsupported or the host resolves into a blocked range
  */
-export async function resolvePublicUrl(rawUrl: string): Promise<URL> {
+export async function resolvePublicUrlAndAddress(
+  rawUrl: string,
+): Promise<{ url: URL; address: string }> {
   let url: URL;
   try {
     url = new URL(rawUrl);
@@ -158,7 +158,7 @@ export async function resolvePublicUrl(rawUrl: string): Promise<URL> {
     if (isBlockedAddress(hostname)) {
       throw new Error(`Refusing to fetch a non-public address: ${hostname}`);
     }
-    return url;
+    return { url, address: hostname };
   }
 
   const addresses = await lookup(hostname, { all: true });
@@ -173,5 +173,27 @@ export async function resolvePublicUrl(rawUrl: string): Promise<URL> {
     );
   }
 
-  return url;
+  return { url, address: addresses[0].address };
+}
+
+/**
+ * Validates a URL that the server is about to fetch on a client's behalf,
+ * rejecting non-HTTP schemes and hosts that resolve into a blocked range.
+ *
+ * This check alone does not close the DNS-rebinding window: a caller that
+ * fetches by hostname after validating here re-resolves at connect time,
+ * so a name that answers publicly now and privately a moment later still
+ * gets through, and the response carries the result back. Callers that
+ * want the window closed must pin the connection with
+ * `requestPinnedToVettedAddress` using the address from
+ * `resolvePublicUrlAndAddress`. `/thumbnail` does; `/page-content` does
+ * not yet, and carries the residual: up to one GET per redirect hop from
+ * inside the instance's network, and the readable text of whatever those
+ * GETs return.
+ *
+ * @returns The parsed URL when it is safe to fetch
+ * @throws When the scheme is unsupported or the host resolves into a blocked range
+ */
+export async function resolvePublicUrl(rawUrl: string): Promise<URL> {
+  return (await resolvePublicUrlAndAddress(rawUrl)).url;
 }

tsconfig.node.json

@@ -9,5 +9,11 @@
     "allowSyntheticDefaultImports": true,
     "strict": true
   },
-  "include": ["vite.config.ts", "server", "shared"]
+  "include": [
+    "vite.config.ts",
+    "server",
+    "shared",
+    "e2e",
+    "playwright.config.ts"
+  ]
 }