backend/evals/heldout_queries.json

{
  "_comment": "Held-out labels for the web-search gate. Deliberately NOT derived from tests/test_web_heuristic.py, which the regex was tuned against and therefore scores 100% on by construction. Phrasings here are casual and varied, the way people actually type. Label = does answering CORRECTLY require information the model cannot have from training.",
  "needs_web": [
    "did the fed cut rates",
    "how much is a gallon of gas in california",
    "is the new dune movie out yet",
    "what's trending on github",
    "who's leading the premier league",
    "any news on the mars mission",
    "how's the stock market doing",
    "what did apple announce",
    "is bitcoin up or down",
    "what's the exchange rate for yen",
    "who signed with the lakers",
    "what happened at the climate summit",
    "latest version of react",
    "has the infrastructure bill passed",
    "who's hosting the oscars",
    "did they announce the new pixel",
    "what's happening with the port strike",
    "is there a new season of severance",
    "how many subscribers does mrbeast have",
    "what's the population of tokyo these days"
  ],
  "no_web": [
    "why is the sky blue",
    "how do i center a div",
    "what's the capital of peru",
    "explain recursion with an example",
    "write a haiku about rain",
    "what's 15% of 340",
    "difference between tcp and udp",
    "how do vaccines work",
    "best practices for naming variables",
    "summarize the plot of hamlet",
    "convert 100 fahrenheit to celsius",
    "what does ROI stand for",
    "how to boil an egg",
    "who wrote pride and prejudice",
    "explain the doppler effect",
    "is python interpreted or compiled",
    "what causes inflation",
    "how many continents are there",
    "give me tips for a job interview",
    "what's the plural of octopus"
  ]
}