Spaces:
Running
Running
| /** | |
| * The golden set for the offline eval: a fixed collection of queries, each with | |
| * a realistic set of candidate results, the subset of those a human would call | |
| * relevant, and (for the answer eval) a reference answer and a rubric. | |
| * | |
| * This is deliberately hand-curated and stable. It is a regression baseline, | |
| * not a benchmark: the point is that a change to the reranker, the prompt, or | |
| * the model moves these numbers in a direction a human can read, not that the | |
| * numbers are absolute. | |
| * | |
| * To grow the set, add an entry with the same shape. Keep the relevant labels | |
| * unambiguous (a result is clearly about the query or clearly not) so the | |
| * metric does not depend on a borderline call. | |
| * | |
| * Retrieval constraint: the input top-3 (the `results` array order) must not | |
| * already hold every relevant result it can fit. If it does, a no-op reranker | |
| * (which preserves input order) scores that entry as well as a working one, | |
| * and the retrieval eval's floors stop being falsifiable. The per-entry | |
| * validator and the "stays falsifiable" aggregate in goldenSet.test.ts pin | |
| * this; re-tune the floors in thresholds.ts if the set changes substantially. | |
| */ | |
| interface GoldenResult { | |
| title: string; | |
| snippet: string; | |
| url: string; | |
| } | |
| export interface GoldenQuery { | |
| /** Stable id used in reports. */ | |
| id: string; | |
| query: string; | |
| results: GoldenResult[]; | |
| /** Indices into `results` a human would consider relevant to `query`. */ | |
| relevant: number[]; | |
| /** Key facts a correct answer must contain (for the LLM judge). */ | |
| referenceAnswer: string; | |
| /** What a good answer does, checked one point at a time by the judge. */ | |
| rubric: string[]; | |
| /** | |
| * The answer's key fact exists only in the snippets, not in parametric | |
| * knowledge or the result titles. These are the entries that fail if the | |
| * search-results path breaks; the answer eval grades them as a group so a | |
| * handful of them can't be drowned out by the rest of the set. | |
| */ | |
| snippetOnly?: boolean; | |
| } | |
| export const goldenQueries: GoldenQuery[] = [ | |
| { | |
| id: "apollo11-first-moonwalker", | |
| query: "Who was the first person to walk on the moon?", | |
| results: [ | |
| { | |
| title: "Apollo 11: NASA's mission that put humans on the Moon", | |
| snippet: | |
| "On July 20, 1969, astronaut Neil Armstrong stepped onto the lunar surface, followed hours later by Buzz Aldrin.", | |
| url: "https://nasa.gov/apollo-11", | |
| }, | |
| { | |
| title: "Neil Armstrong signed baseball cards for sale", | |
| snippet: | |
| "Buy a signed memorabilia card from the astronaut who first walked on the Moon. Ships free.", | |
| url: "https://memorabilia.shop/armstrong-card", | |
| }, | |
| { | |
| title: "Moon Hotels: stay in a suite with a lunar view", | |
| snippet: | |
| "Luxury suites overlooking the crater. Book your once-in-a-lifetime stay on the Moon.", | |
| url: "https://travel.com/moon-hotels", | |
| }, | |
| { | |
| title: "Apollo 11 - Wikipedia", | |
| snippet: | |
| "Apollo 11 was the spaceflight that landed the first humans, Neil Armstrong and Buzz Aldrin, on the Moon on 20 July 1969.", | |
| url: "https://wikipedia.org/wiki/Apollo_11", | |
| }, | |
| { | |
| title: "Moon phase calendar for this month", | |
| snippet: | |
| "Track new moon, full moon and quarter phases with our interactive monthly calendar.", | |
| url: "https://almanac.com/moon-phases", | |
| }, | |
| ], | |
| relevant: [0, 3], | |
| referenceAnswer: | |
| "Neil Armstrong was the first person to walk on the Moon, on 20 July 1969, during the Apollo 11 mission; Buzz Aldrin was the second.", | |
| rubric: [ | |
| "Names Neil Armstrong as the first person on the Moon.", | |
| "Gives the correct date (20 July 1969) or at least the correct mission (Apollo 11).", | |
| "Does not name Buzz Aldrin as the first (Aldrin was second).", | |
| ], | |
| }, | |
| { | |
| id: "css-flexbox-center", | |
| query: "how do I center a div with CSS flexbox", | |
| results: [ | |
| { | |
| title: "A Complete Guide to Flexbox - CSS-Tricks", | |
| snippet: | |
| "Use display: flex with justify-content: center and align-items: center to center content both ways.", | |
| url: "https://css-tricks.com/flexbox-guide", | |
| }, | |
| { | |
| title: "Free CSS Reset stylesheet download", | |
| snippet: | |
| "A minimal reset to normalize browser defaults. Copy, paste, ship.", | |
| url: "https://css-tools.com/reset", | |
| }, | |
| { | |
| title: "What is a div tag in HTML?", | |
| snippet: | |
| "The div element is a generic container for flow content with no required attributes.", | |
| url: "https://html-reference.com/div", | |
| }, | |
| { | |
| title: "Flexbox cheat sheet poster", | |
| snippet: | |
| "Printable poster of all flexbox properties. Order now and master layout at a glance.", | |
| url: "https://printables.com/flexbox-poster", | |
| }, | |
| { | |
| title: "CSS Flexbox - MDN Web Docs", | |
| snippet: | |
| "The flex container's main and cross axes let you center items with justify-content and align-items.", | |
| url: "https://developer.mozilla.org/flexbox", | |
| }, | |
| ], | |
| relevant: [0, 4], | |
| referenceAnswer: | |
| "Set the parent to display: flex, then use justify-content: center (main axis) and align-items: center (cross axis) to center the child both horizontally and vertically.", | |
| rubric: [ | |
| "Mentions setting the container to display: flex.", | |
| "Names justify-content and/or align-items (the actual properties).", | |
| "Does not present a float, margin, or absolute-positioning hack as the flexbox answer.", | |
| ], | |
| }, | |
| { | |
| id: "australia-capital", | |
| query: "Qual é a capital da Austrália?", | |
| results: [ | |
| { | |
| title: "Canberra: capital of Australia - Visit Canberra", | |
| snippet: | |
| "Canberra is the capital city of Australia and the seat of the federal government, in the Australian Capital Territory.", | |
| url: "https://visitcanberra.com.au/", | |
| }, | |
| { | |
| title: "Sydney: top 10 things to do", | |
| snippet: | |
| "From the Opera House to Bondi Beach, here is how to spend a week in Australia's biggest city.", | |
| url: "https://travel.com/sydney-things-to-do", | |
| }, | |
| { | |
| title: "Australian visa: eligibility and how to apply", | |
| snippet: | |
| "Check if you qualify for an Australian visitor, work or student visa and start your application.", | |
| url: "https://immigration.gov/australia-visa", | |
| }, | |
| { | |
| title: "Austrália - Wikipédia", | |
| snippet: | |
| "A capital da Austrália é Camberra, e a maior cidade é Sydney.", | |
| url: "https://pt.wikipedia.org/wiki/Austrália", | |
| }, | |
| ], | |
| relevant: [0, 3], | |
| referenceAnswer: | |
| "A capital da Austrália é Camberra (Canberra). Sydney é a cidade mais populosa, mas não é a capital.", | |
| rubric: [ | |
| "Names Canberra (Camberra) as the capital.", | |
| "Does not name Sydney as the capital.", | |
| "If it clarifies, correctly notes Sydney is the largest city but not the capital.", | |
| "Replies in Portuguese, the language of the question.", | |
| ], | |
| }, | |
| { | |
| id: "git-merge-vs-rebase", | |
| query: "difference between git merge and git rebase", | |
| results: [ | |
| { | |
| title: "Git merge vs rebase - when to use which", | |
| snippet: | |
| "git merge creates a new commit joining two histories; git rebase replays your commits on top of another branch.", | |
| url: "https://git-guides.com/merge-vs-rebase", | |
| }, | |
| { | |
| title: "GitKraken: visual Git client download", | |
| snippet: | |
| "A cross-platform Git GUI with a visual commit graph. Free for personal use.", | |
| url: "https://gitkraken.com/download", | |
| }, | |
| { | |
| title: "Git logo and brand assets", | |
| snippet: | |
| "Download the official Git logo in SVG and PNG for your site or slides.", | |
| url: "https://brandassets.com/git-logo", | |
| }, | |
| { | |
| title: "Git hosting pricing plans", | |
| snippet: | |
| "Compare free, team and enterprise plans for private repositories and CI minutes.", | |
| url: "https://codehost.dev/pricing", | |
| }, | |
| { | |
| title: "git-rebase - Git documentation", | |
| snippet: | |
| "Rebase rewrites your branch's commits to sit on top of the upstream, producing a linear history.", | |
| url: "https://git-scm.com/docs/git-rebase", | |
| }, | |
| ], | |
| relevant: [0, 4], | |
| referenceAnswer: | |
| "git merge combines two branches into a new merge commit, preserving both histories. git rebase replays your commits on top of another branch, rewriting them for a linear history.", | |
| rubric: [ | |
| "Explains that merge creates/joins histories (a merge commit or preserved branch point).", | |
| "Explains that rebase replays/moves commits onto another base (linear history, rewrites).", | |
| "Does not swap the two definitions.", | |
| ], | |
| }, | |
| { | |
| id: "wwii-end-year", | |
| query: "What year did World War II end?", | |
| results: [ | |
| { | |
| title: "World War II - Wikipedia", | |
| snippet: | |
| "World War II was a global conflict that lasted from 1939 to 1945, ending with the surrender of Japan in September 1945.", | |
| url: "https://wikipedia.org/wiki/World_War_II", | |
| }, | |
| { | |
| title: "World War II strategy board game", | |
| snippet: | |
| "Command the armies of 1939-1945 in this award-winning turn-based strategy game.", | |
| url: "https://boardgames.com/wwii-strategy", | |
| }, | |
| { | |
| title: "Vintage WWII poster prints", | |
| snippet: | |
| "Restored wartime posters, ready to frame. A bold piece for any wall.", | |
| url: "https://prints.com/wwii-posters", | |
| }, | |
| { | |
| title: "The end of World War II - History.com", | |
| snippet: | |
| "V-E Day in May 1945 and V-J Day in September 1945 marked the end of the war in Europe and the Pacific.", | |
| url: "https://history.com/wwii-end", | |
| }, | |
| ], | |
| relevant: [0, 3], | |
| referenceAnswer: | |
| "World War II ended in 1945 (Europe in May 1945, and globally with Japan's surrender in September 1945).", | |
| rubric: ["Gives 1945 as the end year.", "Does not give a wrong year."], | |
| }, | |
| { | |
| id: "jwt-secure-rest-api", | |
| query: "how to secure a REST API with JWT", | |
| results: [ | |
| { | |
| title: "JSON Web Tokens: a practical guide - Auth0", | |
| snippet: | |
| "Sign tokens with a strong secret, keep access tokens short-lived, and validate the signature and expiry on every request.", | |
| url: "https://auth0.com/jwt-guide", | |
| }, | |
| { | |
| title: "REST API tutorial for beginners", | |
| snippet: | |
| "Learn to build CRUD endpoints with HTTP verbs, status codes and JSON responses.", | |
| url: "https://tutorials.com/rest-101", | |
| }, | |
| { | |
| title: "JWT logo vector download", | |
| snippet: "The official JWT wordmark as an SVG for your docs.", | |
| url: "https://logos.com/jwt-svg", | |
| }, | |
| { | |
| title: "Managed JWT authentication service pricing", | |
| snippet: | |
| "Start free, then pay per monthly active user for hosted auth.", | |
| url: "https://authvendor.io/pricing", | |
| }, | |
| { | |
| title: "JWT Security Best Practices - OWASP", | |
| snippet: | |
| "Validate alg, nbf and exp claims; never put secrets in the payload; rotate signing keys.", | |
| url: "https://owasp.org/jwt-best-practices", | |
| }, | |
| ], | |
| relevant: [0, 4], | |
| referenceAnswer: | |
| "Use short-lived signed access tokens, validate the signature and claims (exp, nbf, aud) on every request, keep the signing secret strong and rotated, and use refresh tokens for re-authentication.", | |
| rubric: [ | |
| "Mention validating the token signature on the server.", | |
| "Mention expiry / short-lived tokens (or exp).", | |
| "Does not claim the JWT payload is encrypted or secret by default.", | |
| ], | |
| }, | |
| { | |
| id: "largest-ocean", | |
| query: "qual o maior oceano do mundo", | |
| results: [ | |
| { | |
| title: "Oceanos - Wikipédia", | |
| snippet: | |
| "O Oceano Pacífico é o maior e mais profundo oceano da Terra, cobrindo cerca de um terço da superfície.", | |
| url: "https://pt.wikipedia.org/wiki/Oceano_Pacífico", | |
| }, | |
| { | |
| title: "Cruise to the South Pacific: 12 nights", | |
| snippet: | |
| "Sail between island paradises with daily excursions and all-inclusive dining.", | |
| url: "https://cruises.com/south-pacific", | |
| }, | |
| { | |
| title: "Ocean life: meet the marine animals", | |
| snippet: | |
| "From jellyfish to blue whales, explore the creatures that live in the sea.", | |
| url: "https://nature.com/ocean-animals", | |
| }, | |
| { | |
| title: "Geografia: os cinco oceanos do planeta", | |
| snippet: | |
| "O Pacífico é o maior oceano, seguido do Atlântico, Índico, Ártico e Antártico.", | |
| url: "https://geografia.br/os-oceanos", | |
| }, | |
| ], | |
| relevant: [0, 3], | |
| referenceAnswer: "O maior oceano do mundo é o Oceano Pacífico.", | |
| rubric: [ | |
| "Names the Pacific (Pacífico) as the largest ocean.", | |
| "Does not name the Atlantic as the largest.", | |
| "Replies in Portuguese, the language of the question.", | |
| ], | |
| }, | |
| { | |
| id: "hamlet-author", | |
| query: "Who wrote the play Hamlet?", | |
| results: [ | |
| { | |
| title: "Hamlet - Wikipedia", | |
| snippet: | |
| "Hamlet is a tragedy written by William Shakespeare between 1599 and 1601, one of his most performed works.", | |
| url: "https://wikipedia.org/wiki/Hamlet", | |
| }, | |
| { | |
| title: "Hamlet at the Royal Court: tickets", | |
| snippet: | |
| "Book seats for the season's production of Shakespeare's Hamlet. Premium balcony available.", | |
| url: "https://theatre.com/hamlet-tickets", | |
| }, | |
| { | |
| title: "What is a hamlet (settlement)?", | |
| snippet: | |
| "A hamlet is a small rural settlement, typically smaller than a village and without a church.", | |
| url: "https://geography.com/hamlet-settlement", | |
| }, | |
| { | |
| title: "William Shakespeare: the Bard of Avon", | |
| snippet: | |
| "Shakespeare wrote 39 plays including Hamlet, Macbeth and Othello, defining the English language stage.", | |
| url: "https://bard.org/william-shakespeare", | |
| }, | |
| ], | |
| relevant: [0, 3], | |
| referenceAnswer: "Hamlet was written by William Shakespeare.", | |
| rubric: [ | |
| "Names William Shakespeare as the author.", | |
| "Does not confuse the play with the meaning of 'hamlet' (a settlement).", | |
| ], | |
| }, | |
| { | |
| id: "react-useeffect-infinite-loop", | |
| query: "react useEffect infinite loop cause", | |
| results: [ | |
| { | |
| title: "React useEffect: avoiding the infinite render loop", | |
| snippet: | |
| "A new object or function in the dependency array re-triggers the effect every render. Stabilize it or drop it from the deps.", | |
| url: "https://react-patterns.com/useeffect-loop", | |
| }, | |
| { | |
| title: "React logo and brand guidelines", | |
| snippet: "Use the official React atom logo in your slides and docs.", | |
| url: "https://reactjs.org/brand", | |
| }, | |
| { | |
| title: "React Native: build mobile apps", | |
| snippet: | |
| "Use the latest JavaScript to build native iOS and Android apps with a single codebase.", | |
| url: "https://reactnative.dev/", | |
| }, | |
| { | |
| title: "Frontend course: React pricing", | |
| snippet: | |
| "Lifetime access to 40 hours of React and TypeScript video lessons.", | |
| url: "https://course.dev/react-pricing", | |
| }, | |
| { | |
| title: "Why does my useEffect run twice? - Stack Overflow", | |
| snippet: | |
| "Updating state that is in the dependency array (or returning a new reference each render) re-runs the effect, which updates state again.", | |
| url: "https://stackoverflow.com/q/useeffect-twice", | |
| }, | |
| ], | |
| relevant: [0, 4], | |
| referenceAnswer: | |
| "A useEffect infinite loop is usually caused by a dependency that changes on every render (a new object/function reference) or by updating state that is itself in the dependency array, which re-runs the effect, which updates the state again. Stabilize the dependency (useMemo/useCallback) or restructure the effect.", | |
| rubric: [ | |
| "Identifies a changing/incorrect dependency array as the cause.", | |
| "Mentions state updates that re-trigger the effect (the feedback loop).", | |
| "Suggests a real fix (stabilize the dep, or restructure), not just 'add a key'.", | |
| ], | |
| }, | |
| { | |
| id: "water-boiling-point", | |
| query: "What is the boiling point of water at sea level?", | |
| results: [ | |
| { | |
| title: "Boiling point - Wikipedia", | |
| snippet: | |
| "The boiling point of water is 100 °C (212 °F) at standard atmospheric pressure (sea level).", | |
| url: "https://wikipedia.org/wiki/Boiling_point", | |
| }, | |
| { | |
| title: "Electric kettle: 1.7L fast boil", | |
| snippet: | |
| "Boil a kettle of water in three minutes with rapid heat and an auto shut-off.", | |
| url: "https://appliance.com/electric-kettle", | |
| }, | |
| { | |
| title: "Industrial water boiler for process heat", | |
| snippet: | |
| "Steam generators rated to 15 bar for continuous industrial use.", | |
| url: "https://industrial.com/water-boiler", | |
| }, | |
| { | |
| title: "Water properties reference - Chemistry LibreTexts", | |
| snippet: | |
| "At 1 atm, water boils at 100 °C; the point drops with altitude as pressure falls.", | |
| url: "https://chem.libretexts.com/water-properties", | |
| }, | |
| ], | |
| relevant: [0, 3], | |
| referenceAnswer: "At sea level (1 atm), water boils at 100 °C (212 °F).", | |
| rubric: [ | |
| "Gives 100 °C or 212 °F.", | |
| "Ties it to sea level / standard pressure (or notes it changes with altitude).", | |
| ], | |
| }, | |
| { | |
| id: "magnus-effect", | |
| query: "what causes the curve of a spinning football (soccer)", | |
| results: [ | |
| { | |
| title: "The Magnus effect explained", | |
| snippet: | |
| "A spinning ball drags air around it, creating a pressure difference that pushes the ball sideways, curving its path.", | |
| url: "https://physics-now.com/magnus-effect", | |
| }, | |
| { | |
| title: "Football boots: 2026 lineup", | |
| snippet: | |
| "Lightweight studs and a textured surface for grip and control on any pitch.", | |
| url: "https://sports.com/football-boots", | |
| }, | |
| { | |
| title: "Football (soccer) rules overview", | |
| snippet: | |
| "The basic laws of the game: offside, fouls, corners and how a match is structured.", | |
| url: "https://rules.com/soccer-laws", | |
| }, | |
| { | |
| title: "Why free kicks bend - BBC Sport", | |
| snippet: | |
| "Striking the ball off-centre spins it; the resulting Magnus effect is what bends a free kick around the wall.", | |
| url: "https://bbc.com/sport/free-kicks", | |
| }, | |
| ], | |
| relevant: [0, 3], | |
| referenceAnswer: | |
| "The curve is caused by the Magnus effect: a spinning ball creates a pressure difference in the surrounding air that pushes it sideways, bending its trajectory.", | |
| rubric: [ | |
| "Names the Magnus effect or the underlying pressure difference from spin.", | |
| "Does not attribute the curve to air 'pushing' it in a single direction with no spin mechanism.", | |
| ], | |
| }, | |
| { | |
| id: "photosynthesis-equation", | |
| query: "what is the chemical equation for photosynthesis", | |
| results: [ | |
| { | |
| title: "Photosynthesis - an overview", | |
| snippet: | |
| "6 CO2 + 6 H2O + light -> C6H12O6 + 6 O2: plants convert carbon dioxide and water into glucose and oxygen.", | |
| url: "https://biology.com/photosynthesis", | |
| }, | |
| { | |
| title: "Garden sunlight: how much does your plant need?", | |
| snippet: | |
| "Most houseplants do well with a few hours of indirect light a day.", | |
| url: "https://gardening.com/plant-light", | |
| }, | |
| { | |
| title: "Succulent care 101", | |
| snippet: | |
| "Water less often, use a gritty mix, and give bright indirect light.", | |
| url: "https://plants101.com/succulents", | |
| }, | |
| { | |
| title: "The equation of photosynthesis - Khan Academy", | |
| snippet: | |
| "Carbon dioxide plus water, using light energy, yields glucose and oxygen.", | |
| url: "https://khanacademy.org/photosynthesis-equation", | |
| }, | |
| ], | |
| relevant: [0, 3], | |
| referenceAnswer: | |
| "6 CO2 + 6 H2O + light energy -> C6H12O6 (glucose) + 6 O2.", | |
| rubric: [ | |
| "Includes CO2 and H2O as reactants and glucose (C6H12O6) and O2 as products.", | |
| "Mentions light (energy) as a requirement.", | |
| ], | |
| }, | |
| { | |
| id: "speed-of-light", | |
| query: "What is the speed of light in a vacuum?", | |
| results: [ | |
| { | |
| title: "Speed of light - Wikipedia", | |
| snippet: | |
| "The speed of light in vacuum, usually denoted c, is exactly 299,792,458 metres per second.", | |
| url: "https://wikipedia.org/wiki/Speed_of_light", | |
| }, | |
| { | |
| title: "LED light bulbs: energy-saving 6-pack", | |
| snippet: | |
| "Bright, long-lasting LED bulbs for your home. Save on your electricity bill.", | |
| url: "https://home.com/led-bulbs", | |
| }, | |
| { | |
| title: "Fiber internet: gigabit 'speed of light' plans", | |
| snippet: | |
| "Blazing-fast fiber up to 1 Gbps. Marketing calls it the speed of light.", | |
| url: "https://isp.com/fiber-gigabit", | |
| }, | |
| { | |
| title: "The constant c in physics - HyperPhysics", | |
| snippet: | |
| "c = 299792458 m/s is the invariant speed that sets the maximum speed of causality.", | |
| url: "https://hyperphysics.com/speed-of-light", | |
| }, | |
| ], | |
| relevant: [0, 3], | |
| referenceAnswer: | |
| "The speed of light in a vacuum is exactly 299,792,458 metres per second (about 3.00 x 10^8 m/s).", | |
| rubric: [ | |
| "Gives 299,792,458 m/s or about 3.00 x 10^8 m/s.", | |
| "Does not give the speed of light in air or glass as the vacuum value.", | |
| ], | |
| }, | |
| { | |
| id: "mona-lisa-painter", | |
| query: "Who painted the Mona Lisa?", | |
| results: [ | |
| { | |
| title: "Mona Lisa - The Louvre", | |
| snippet: | |
| "La Joconde, by Leonardo da Vinci, painted in the early 16th century, is the museum's most visited work.", | |
| url: "https://louvre.fr/mona-lisa", | |
| }, | |
| { | |
| title: "Mona Lisa Halloween masks and costumes", | |
| snippet: | |
| "Be the enigma this Halloween with our best-selling Mona Lisa mask.", | |
| url: "https://costumes.com/mona-lisa-mask", | |
| }, | |
| { | |
| title: "Mona Lisa (singer): discography", | |
| snippet: | |
| "A French singer-songwriter's albums, singles and touring dates.", | |
| url: "https://music.com/mona-lisa-singer", | |
| }, | |
| { | |
| title: "Mona Lisa - Wikipedia", | |
| snippet: | |
| "The Mona Lisa is a portrait painting by the Italian Renaissance artist Leonardo da Vinci.", | |
| url: "https://wikipedia.org/wiki/Mona_Lisa", | |
| }, | |
| ], | |
| relevant: [0, 3], | |
| referenceAnswer: "The Mona Lisa was painted by Leonardo da Vinci.", | |
| rubric: [ | |
| "Names Leonardo da Vinci as the painter.", | |
| "Does not confuse the painting with a person or artist of the same name.", | |
| ], | |
| }, | |
| { | |
| id: "baking-soda-vs-powder", | |
| query: "difference between baking soda and baking powder", | |
| results: [ | |
| { | |
| title: "Baking soda vs baking powder, explained", | |
| snippet: | |
| "Baking soda is pure sodium bicarbonate and needs an acid to react; baking powder already has an acid added.", | |
| url: "https://cooking-science.com/soda-vs-powder", | |
| }, | |
| { | |
| title: "Arm & Hammer baking soda, 4 lb", | |
| snippet: | |
| "Multipurpose baking soda for baking, cleaning and deodorizing.", | |
| url: "https://grocery.com/baking-soda-4lb", | |
| }, | |
| { | |
| title: "Classic lemon drizzle cake recipe", | |
| snippet: "A moist lemon cake with a sweet drizzle topping. Serves 10.", | |
| url: "https://recipes.com/lemon-drizzle", | |
| }, | |
| { | |
| title: "What is the difference? - wikiHow", | |
| snippet: | |
| "Baking powder contains baking soda plus an acid and starch, so it leavens on its own.", | |
| url: "https://wikihow.com/baking-soda-vs-powder", | |
| }, | |
| ], | |
| relevant: [0, 3], | |
| referenceAnswer: | |
| "Baking soda is sodium bicarbonate alone and needs an acid to leaven; baking powder is baking soda plus a built-in acid (and usually starch), so it leavens by itself.", | |
| rubric: [ | |
| "Notes baking soda needs an acid to react.", | |
| "Notes baking powder already contains an acid (works on its own).", | |
| ], | |
| }, | |
| { | |
| id: "tallest-mountain", | |
| query: "What is the tallest mountain in the world?", | |
| results: [ | |
| { | |
| title: "Mount Everest - Wikipedia", | |
| snippet: | |
| "Everest, at 8,849 m, is Earth's highest mountain above sea level, on the Nepal-Tibet border.", | |
| url: "https://wikipedia.org/wiki/Mount_Everest", | |
| }, | |
| { | |
| title: "Everest base camp trekking tours", | |
| snippet: | |
| "14-day guided trek to Everest base camp. Flights and guides included.", | |
| url: "https://treks.com/everest-base-camp", | |
| }, | |
| { | |
| title: "Mountain bike sale: full-suspension", | |
| snippet: "Trail-ready full-suspension mountain bikes, from $699.", | |
| url: "https://cycles.com/mountain-bike", | |
| }, | |
| { | |
| title: "Highest peaks on Earth - Britannica", | |
| snippet: | |
| "Mount Everest is the highest mountain above sea level at 8,848.86 m.", | |
| url: "https://britannica.com/highest-peaks", | |
| }, | |
| ], | |
| relevant: [0, 3], | |
| referenceAnswer: | |
| "Mount Everest is the tallest mountain in the world (highest above sea level), at about 8,849 m.", | |
| rubric: [ | |
| "Names Mount Everest as the tallest mountain.", | |
| "Gives an elevation near 8,849 m (or 8,848 m), or at least identifies Everest correctly.", | |
| ], | |
| }, | |
| { | |
| id: "change-car-tire", | |
| query: "how to change a flat car tire", | |
| results: [ | |
| { | |
| title: "How to Change a Flat Tire - AAA", | |
| snippet: | |
| "Park safely, put on the handbrake, loosen the lug nuts, jack up the car, and swap in the spare.", | |
| url: "https://aaa.com/change-a-flat", | |
| }, | |
| { | |
| title: "All-season tires: price comparison", | |
| snippet: "Compare prices for popular all-season tires in your size.", | |
| url: "https://tires.com/all-season-prices", | |
| }, | |
| { | |
| title: "Alloy wheel accessories and caps", | |
| snippet: | |
| "Decorative wheel covers and valve caps to finish off your rims.", | |
| url: "https://auto.com/wheel-caps", | |
| }, | |
| { | |
| title: "Spare tire change step by step - YouTube", | |
| snippet: | |
| "A 3-minute video walking through jacking, swapping and tightening the lug nuts.", | |
| url: "https://youtube.com/watch/change-flat", | |
| }, | |
| ], | |
| relevant: [0, 3], | |
| referenceAnswer: | |
| "Park on level ground, set the handbrake, loosen the lug nuts, jack up the car, remove the flat, fit the spare, tighten the nuts, and lower the car.", | |
| rubric: [ | |
| "Mentions jacking the car up.", | |
| "Mentions loosening/removing the lug nuts and fitting the spare.", | |
| "Mentions a safety step (handbrake / parking on level ground).", | |
| ], | |
| }, | |
| { | |
| id: "first-iphone-year", | |
| query: "What year did the first iPhone release?", | |
| results: [ | |
| { | |
| title: "iPhone - Wikipedia", | |
| snippet: | |
| "The original iPhone was announced in January 2007 and released in the United States on June 29, 2007.", | |
| url: "https://wikipedia.org/wiki/IPhone", | |
| }, | |
| { | |
| title: "iPhone 16 Pro review: is it worth it?", | |
| snippet: | |
| "Our full review of the latest flagship, two years after its launch.", | |
| url: "https://techreview.com/iphone-16-pro", | |
| }, | |
| { | |
| title: "iPhone cases and screen protectors", | |
| snippet: | |
| "Shop rugged cases and tempered-glass protectors for every iPhone model.", | |
| url: "https://cases.com/iphone", | |
| }, | |
| { | |
| title: "Apple unveils iPhone - Apple Newsroom (2007)", | |
| snippet: | |
| "Today Apple reinvented the phone. The iPhone combines three revolutionary devices.", | |
| url: "https://apple.com/newsroom/2007-iphone", | |
| }, | |
| ], | |
| relevant: [0, 3], | |
| referenceAnswer: | |
| "The first iPhone was released in 2007 (announced January 2007, released June 29, 2007).", | |
| rubric: [ | |
| "Gives 2007 as the release year.", | |
| "Does not give a different year.", | |
| ], | |
| }, | |
| { | |
| id: "reset-wifi-router", | |
| query: "how do I reset my Wi-Fi router to factory settings", | |
| results: [ | |
| { | |
| title: "Factory reset your router - ISP support", | |
| snippet: | |
| "Press and hold the recessed reset button for about 10 seconds until the lights blink.", | |
| url: "https://isp.com/factory-reset-router", | |
| }, | |
| { | |
| title: "Mesh Wi-Fi system review: 3-pack", | |
| snippet: | |
| "Whole-home coverage from three nodes. Our verdict after a month.", | |
| url: "https://techreview.com/mesh-wifi", | |
| }, | |
| { | |
| title: "Cable modem and router bundles", | |
| snippet: "Save with a combo modem-router for your home internet.", | |
| url: "https://isp.com/modem-router-bundle", | |
| }, | |
| { | |
| title: "How to factory reset any router - CNET", | |
| snippet: | |
| "Find the reset pinhole, hold it with a paperclip until the status light flashes, and reconfigure.", | |
| url: "https://cnet.com/router-factory-reset", | |
| }, | |
| ], | |
| relevant: [0, 3], | |
| referenceAnswer: | |
| "Press and hold the router's recessed reset button (with a paperclip) for about 10 seconds until the lights blink, which restores factory defaults.", | |
| rubric: [ | |
| "Mentions the physical reset button (pinhole / recessed).", | |
| "Mentions holding it for several (about 10) seconds.", | |
| "Notes that it restores factory defaults / requires reconfiguration.", | |
| ], | |
| }, | |
| { | |
| id: "gold-chemical-symbol", | |
| query: "What is the chemical symbol for gold?", | |
| results: [ | |
| { | |
| title: "Gold - Wikipedia", | |
| snippet: | |
| "Gold is a chemical element with symbol Au and atomic number 79.", | |
| url: "https://wikipedia.org/wiki/Gold", | |
| }, | |
| { | |
| title: "14k gold jewelry: rings and chains", | |
| snippet: "Shop fine 14k gold jewelry. Free shipping on all orders.", | |
| url: "https://jewelry.com/gold-rings", | |
| }, | |
| { | |
| title: "Gold Rewards credit card benefits", | |
| snippet: "Earn points on every purchase with the Gold Rewards card.", | |
| url: "https://bank.com/gold-card", | |
| }, | |
| { | |
| title: "The Periodic Table: Au (gold)", | |
| snippet: | |
| "Au, atomic number 79, a transition metal widely used in electronics.", | |
| url: "https://periodic-table.com/au", | |
| }, | |
| ], | |
| relevant: [0, 3], | |
| referenceAnswer: | |
| "The chemical symbol for gold is Au (from the Latin 'aurum').", | |
| rubric: [ | |
| "Gives Au as the chemical symbol for gold.", | |
| "Does not give the symbol for a different element.", | |
| ], | |
| }, | |
| { | |
| id: "remove-stain-white-shirt", | |
| query: "how to remove a stain from a white shirt", | |
| results: [ | |
| { | |
| title: "Stains: A History of Dirt and Cleanliness", | |
| snippet: | |
| "A cultural history of why we care about clean clothes. Hardcover, 320 pages.", | |
| url: "https://books.com/stains-history", | |
| }, | |
| { | |
| title: "How to Remove Common Stains from Clothes - wikiHow", | |
| snippet: | |
| "Treat ink, wine, grass and grease stains with the right solvent before washing.", | |
| url: "https://wikihow.com/remove-clothing-stains", | |
| }, | |
| { | |
| title: "Dry cleaning services near you", | |
| snippet: "Book a same-week dry clean and press for your garments.", | |
| url: "https://cleaners.com/near-you", | |
| }, | |
| { | |
| title: "Removing ink, wine, and oil from white fabric", | |
| snippet: | |
| "Step-by-step for white shirts: dab, don't rub, and pre-treat before the wash.", | |
| url: "https://fabriccare.com/white-shirt-stains", | |
| }, | |
| ], | |
| relevant: [1, 3], | |
| referenceAnswer: | |
| "Dab (don't rub) the stain, pre-treat it with a suitable solvent or stain remover, then wash; for white cotton you can also use oxygen bleach.", | |
| rubric: [ | |
| "Recommends treating/pre-treating the stain before washing.", | |
| "Gives a concrete method (dabbing, a solvent/remover, or oxygen bleach for whites).", | |
| ], | |
| }, | |
| { | |
| id: "capital-of-canada", | |
| query: "what is the capital of canada", | |
| results: [ | |
| { | |
| title: "Canada: the ultimate travel guide", | |
| snippet: "Top 20 places to visit, from the Rockies to the East Coast.", | |
| url: "https://travel.com/canada-guide", | |
| }, | |
| { | |
| title: "Capital of Canada - Wikipedia", | |
| snippet: "Ottawa is the capital city of Canada.", | |
| url: "https://wikipedia.org/wiki/Capital_of_Canada", | |
| }, | |
| { | |
| title: "Apply for a Canadian passport", | |
| snippet: "Start or renew your passport application online.", | |
| url: "https://canada.ca/passport", | |
| }, | |
| { | |
| title: "Ottawa, the capital of Canada", | |
| snippet: "Ottawa, in Ontario, has been Canada's capital since 1859.", | |
| url: "https://ottawa.ca/capital", | |
| }, | |
| ], | |
| relevant: [1, 3], | |
| referenceAnswer: "The capital of Canada is Ottawa.", | |
| rubric: [ | |
| "Names Ottawa as the capital of Canada.", | |
| "Does not name a non-capital Canadian city (e.g. Toronto or Vancouver).", | |
| ], | |
| }, | |
| { | |
| id: "budget-laptop-students", | |
| query: "best budget laptop for students 2024", | |
| results: [ | |
| { | |
| title: "Laptop repair and parts store", | |
| snippet: | |
| "Fix your broken laptop. Screens, keyboards and batteries in stock.", | |
| url: "https://repairs.com/laptop-parts", | |
| }, | |
| { | |
| title: "Best budget laptops for students in 2024", | |
| snippet: | |
| "Our top affordable picks for note-taking and streaming, under $600.", | |
| url: "https://techreview.com/budget-laptops-students", | |
| }, | |
| { | |
| title: "Laptop insurance plans compared", | |
| snippet: | |
| "Protect your device against accidental damage. From $9 a month.", | |
| url: "https://insurance.com/laptop", | |
| }, | |
| { | |
| title: "Top 5 affordable laptops for college", | |
| snippet: | |
| "Long battery life and a comfortable keyboard matter most to students.", | |
| url: "https://collegegear.com/affordable-laptops", | |
| }, | |
| ], | |
| relevant: [1, 3], | |
| referenceAnswer: | |
| "For students on a budget in 2024, pick a sub-$600 laptop with at least 8 GB of RAM, an SSD, and long battery life, as in the budget-student roundups.", | |
| rubric: [ | |
| "Recommends a budget/affordable laptop or gives budget-laptop criteria (price, RAM, battery).", | |
| "Does not answer with a repair service or an insurance plan.", | |
| ], | |
| }, | |
| { | |
| id: "halcyon-robotics-founding", | |
| query: "When was the Halcyon Robotics Institute founded?", | |
| results: [ | |
| { | |
| title: "Halcyon Robotics Institute - About Us", | |
| snippet: | |
| "Founded in 1987, the Halcyon Robotics Institute develops autonomous systems and robotics research.", | |
| url: "https://halcyonrobotics.org/about", | |
| }, | |
| { | |
| title: "Robotics career fair 2024", | |
| snippet: | |
| "Meet employers in robotics and AI. Register for the fall career fair.", | |
| url: "https://careers.com/robotics-fair", | |
| }, | |
| { | |
| title: "Robot building kits for kids", | |
| snippet: "Snap-together robot kits, ages 8 and up. Ships free.", | |
| url: "https://toys.com/robot-kits", | |
| }, | |
| { | |
| title: "Halcyon Robotics Institute - Wikipedia", | |
| snippet: | |
| "The Halcyon Robotics Institute (HRI) is a research organization founded in 1987.", | |
| url: "https://wikipedia.org/wiki/Halcyon_Robotics_Institute", | |
| }, | |
| ], | |
| relevant: [0, 3], | |
| referenceAnswer: "The Halcyon Robotics Institute was founded in 1987.", | |
| rubric: [ | |
| "States 1987 as the founding year.", | |
| "Does not give a different year or say it is unknown.", | |
| ], | |
| snippetOnly: true, | |
| }, | |
| { | |
| id: "nimbus-t3-battery", | |
| query: "how many hours of battery life does the Nimbus T3 laptop have", | |
| results: [ | |
| { | |
| title: "Nimbus T3 review: all-day and then some", | |
| snippet: | |
| "The Nimbus T3 gets 22 hours of battery life on a single charge in our tests.", | |
| url: "https://techreview.com/nimbus-t3", | |
| }, | |
| { | |
| title: "Laptop stands and cooling pads", | |
| snippet: "Ergonomic aluminum laptop stands, from $29.", | |
| url: "https://gadgets.com/laptop-stands", | |
| }, | |
| { | |
| title: "USB-C chargers and adapters", | |
| snippet: "65W and 100W GaN chargers for laptops and phones.", | |
| url: "https://gadgets.com/usb-c-chargers", | |
| }, | |
| { | |
| title: "Nimbus T3 specifications", | |
| snippet: | |
| "Battery: up to 22 hours of video playback on a single charge.", | |
| url: "https://nimbuspc.com/t3-specs", | |
| }, | |
| ], | |
| relevant: [0, 3], | |
| referenceAnswer: | |
| "The Nimbus T3 laptop has up to 22 hours of battery life on a single charge.", | |
| rubric: [ | |
| "States 22 hours of battery life.", | |
| "Does not give a different battery figure.", | |
| ], | |
| snippetOnly: true, | |
| }, | |
| { | |
| id: "gale-7-drone-payload", | |
| query: "what is the maximum payload of the Gale-7 survey drone", | |
| results: [ | |
| { | |
| title: "Gale-7 specifications", | |
| snippet: "Maximum payload 6.2 kg; flight time up to 40 minutes.", | |
| url: "https://galeaero.com/gale-7-specs", | |
| }, | |
| { | |
| title: "Drone insurance and registration", | |
| snippet: | |
| "Register your drone and get liability coverage from $15 a month.", | |
| url: "https://drones.com/insurance", | |
| }, | |
| { | |
| title: "Camera gimbals for drones", | |
| snippet: "3-axis gimbals for 4K cameras, from $199.", | |
| url: "https://dronegear.com/gimbals", | |
| }, | |
| { | |
| title: "Gale-7 review: built for surveying", | |
| snippet: "The Gale-7 carries up to 6.2 kg of survey equipment.", | |
| url: "https://geospatial.com/gale-7-review", | |
| }, | |
| ], | |
| relevant: [0, 3], | |
| referenceAnswer: | |
| "The Gale-7 survey drone carries a maximum payload of 6.2 kg.", | |
| rubric: [ | |
| "States 6.2 kg as the maximum payload.", | |
| "Does not give a different payload figure.", | |
| ], | |
| snippetOnly: true, | |
| }, | |
| ]; | |