Spaces:
Running
Running
| import { describe, expect, it } from "vitest"; | |
| import { goldenQueries } from "./goldenSet.ts"; | |
| import { ndcgAtK, recallAtK } from "./metrics.ts"; | |
| import { | |
| K, | |
| MAX_NOOP_MEAN_NDCG, | |
| MAX_NOOP_MEAN_RECALL, | |
| MIN_MEAN_NDCG, | |
| MIN_MEAN_RECALL, | |
| } from "./thresholds.ts"; | |
| /** | |
| * Bumped deliberately whenever the golden set grows, so an edit to the set | |
| * forces the author to re-read (and re-tune if needed) the thresholds in | |
| * thresholds.ts. | |
| */ | |
| const EXPECTED_QUERY_COUNT = 26; | |
| // Sequences that String.replace treats as special. The app's getSystemPrompt | |
| // substitutes the search results into the prompt via String.replace, and | |
| // getFormattedSearchResults puts the title, url, and snippet of every result | |
| // into that replacement string. A field containing one of these would silently | |
| // corrupt the prompt (a pre-existing bug in client/modules/systemPrompt.ts). | |
| // Rejecting them here keeps the eval from ever grading a corrupted prompt. | |
| const REPLACE_SPECIAL_SEQUENCES = ["$&", "$`", "$'", "$$"]; | |
| /** Structural checks on the golden set so a bad entry fails loudly. */ | |
| function validateGoldenSet() { | |
| const seenIds = new Set<string>(); | |
| for (const g of goldenQueries) { | |
| const urls = g.results.map((r) => r.url); | |
| const duplicate = urls.find((u, i) => urls.indexOf(u) !== i); | |
| expect( | |
| duplicate, | |
| `${g.id}: duplicate result url ${duplicate} (breaks the nDCG <= 1 bound)`, | |
| ).toBeUndefined(); | |
| expect(g.query.length, `${g.id}: empty query`).toBeGreaterThan(0); | |
| expect(g.rubric.length, `${g.id}: empty rubric`).toBeGreaterThan(0); | |
| expect( | |
| g.referenceAnswer.length, | |
| `${g.id}: empty referenceAnswer (weakens the judge)`, | |
| ).toBeGreaterThan(0); | |
| expect(g.relevant.length, `${g.id}: no relevant labels`).toBeGreaterThan(0); | |
| expect( | |
| new Set(g.relevant).size === g.relevant.length, | |
| `${g.id}: duplicate relevant index (dedupes to a trivial entry)`, | |
| ).toBe(true); | |
| for (const i of g.relevant) { | |
| expect( | |
| Number.isInteger(i), | |
| `${g.id}: relevant index ${i} is not an integer`, | |
| ).toBe(true); | |
| expect( | |
| i >= 0 && i < g.results.length, | |
| `${g.id}: relevant index ${i} out of range (0..${g.results.length - 1})`, | |
| ).toBe(true); | |
| } | |
| // Retrieval falsifiability, per entry: the input top-K must not already | |
| // hold every relevant result it can fit, or a no-op reranker (which | |
| // preserves input order) scores this entry as well as a working one. | |
| // Comparing against min(K, relevant.length) rather than "some index >= K" | |
| // also covers entries with more than K labels, where the input top-K can | |
| // be saturated while a label still sits outside it. This is the per-entry | |
| // form of the "stays falsifiable" aggregate below; the aggregate alone | |
| // would let a single degenerate entry pass silently. | |
| const relevantInTopK = g.relevant.filter((i) => i < K).length; | |
| expect( | |
| relevantInTopK < Math.min(K, g.relevant.length), | |
| `${g.id}: the input top-${K} already holds every relevant result it can fit, so a no-op reranker already scores it as well as a working one`, | |
| ).toBe(true); | |
| // The answer eval builds the prompt from these results, but the app slices | |
| // to searchResultsToConsider (6) before sending it. A longer entry would | |
| // grade a prompt the app can never send. | |
| expect( | |
| g.results.length <= 6, | |
| `${g.id}: ${g.results.length} results exceeds the app's 6-result prompt budget`, | |
| ).toBe(true); | |
| for (const r of g.results) { | |
| expect(r.title.length, `${g.id}: empty result title`).toBeGreaterThan(0); | |
| expect(r.snippet.length, `${g.id}: empty result snippet`).toBeGreaterThan( | |
| 0, | |
| ); | |
| expect(r.url.length, `${g.id}: empty result url`).toBeGreaterThan(0); | |
| } | |
| for (const special of REPLACE_SPECIAL_SEQUENCES) { | |
| for (const r of g.results) { | |
| for (const field of [r.title, r.snippet, r.url]) { | |
| expect( | |
| field.includes(special), | |
| `${g.id}: a result field contains ${JSON.stringify(special)} (String.replace would corrupt the prompt)`, | |
| ).toBe(false); | |
| } | |
| } | |
| } | |
| expect(seenIds.has(g.id), `${g.id}: duplicate golden id`).toBe(false); | |
| seenIds.add(g.id); | |
| } | |
| } | |
| describe("golden set", () => { | |
| it("is well-formed", () => { | |
| validateGoldenSet(); | |
| }); | |
| it("has the expected number of queries", () => { | |
| expect(goldenQueries).toHaveLength(EXPECTED_QUERY_COUNT); | |
| }); | |
| it("keeps the no-op ceilings below the eval's floors", () => { | |
| // If a ceiling reached or passed its floor, a no-op reranker would clear | |
| // the floor and the retrieval eval would stop being falsifiable. | |
| expect(MAX_NOOP_MEAN_NDCG).toBeLessThan(MIN_MEAN_NDCG); | |
| expect(MAX_NOOP_MEAN_RECALL).toBeLessThan(MIN_MEAN_RECALL); | |
| }); | |
| it("stays falsifiable: a no-op reranker scores below the eval's floors", () => { | |
| // Score the golden set as a reranker that just preserves the input order | |
| // (the failure mode the retrieval eval exists to catch). If this mean | |
| // climbs to the eval's floors, the set no longer distinguishes a working | |
| // reranker from a dead one - usually because an entry keeps its relevant | |
| // results in the input top-K. | |
| const mean = (xs: number[]) => xs.reduce((a, b) => a + b, 0) / xs.length; | |
| const scored = goldenQueries.map((g) => { | |
| const urls = g.results.map((r) => r.url); | |
| const relevant = g.relevant.map((i) => urls[i]); | |
| return { | |
| ndcg: ndcgAtK(urls, relevant, K), | |
| recall: recallAtK(urls, relevant, K), | |
| }; | |
| }); | |
| expect( | |
| mean(scored.map((s) => s.ndcg)), | |
| `a golden entry keeps its relevant results in the input top-${K}`, | |
| ).toBeLessThanOrEqual(MAX_NOOP_MEAN_NDCG); | |
| expect( | |
| mean(scored.map((s) => s.recall)), | |
| `a golden entry keeps its relevant results in the input top-${K}`, | |
| ).toBeLessThanOrEqual(MAX_NOOP_MEAN_RECALL); | |
| }); | |
| }); | |