{
  "name": "Chinook demo benchmark",
  "description": "Curated NL→SQL questions on the Chinook music store schema. Designed as a realistic 'business analyst workload' rather than the hard-mode BIRD edge cases. Target accuracy ≥ 90% — these are the kinds of questions a product would actually serve.",
  "db_id": "chinook",
  "dialect": "sqlite",
  "questions": [
    {
      "id": "Q01",
      "category": "count",
      "difficulty": "easy",
      "question": "How many albums are in the catalog?",
      "gold_sql": "SELECT COUNT(*) FROM Album"
    },
    {
      "id": "Q02",
      "category": "count",
      "difficulty": "easy",
      "question": "How many tracks are there in total?",
      "gold_sql": "SELECT COUNT(*) FROM Track"
    },
    {
      "id": "Q03",
      "category": "count",
      "difficulty": "easy",
      "question": "How many customers do we have?",
      "gold_sql": "SELECT COUNT(*) FROM Customer"
    },
    {
      "id": "Q04",
      "category": "list",
      "difficulty": "easy",
      "question": "List the names of all genres.",
      "gold_sql": "SELECT Name FROM Genre"
    },
    {
      "id": "Q05",
      "category": "list",
      "difficulty": "easy",
      "question": "List all media types.",
      "gold_sql": "SELECT Name FROM MediaType"
    },
    {
      "id": "Q06",
      "category": "filter",
      "difficulty": "easy",
      "question": "Which customers are from Germany?",
      "gold_sql": "SELECT FirstName, LastName FROM Customer WHERE Country = 'Germany'"
    },
    {
      "id": "Q07",
      "category": "filter",
      "difficulty": "easy",
      "question": "List the customers from Brazil.",
      "gold_sql": "SELECT FirstName, LastName FROM Customer WHERE Country = 'Brazil'"
    },
    {
      "id": "Q08",
      "category": "aggregation",
      "difficulty": "easy",
      "question": "What is the average track length in milliseconds?",
      "gold_sql": "SELECT AVG(Milliseconds) FROM Track"
    },
    {
      "id": "Q09",
      "category": "aggregation",
      "difficulty": "easy",
      "question": "What is the total revenue across all invoices?",
      "gold_sql": "SELECT SUM(Total) FROM Invoice"
    },
    {
      "id": "Q10",
      "category": "aggregation",
      "difficulty": "easy",
      "question": "What is the most expensive track price?",
      "gold_sql": "SELECT MAX(UnitPrice) FROM Track"
    },
    {
      "id": "Q11",
      "category": "join-2",
      "difficulty": "moderate",
      "question": "List all album titles by the artist AC/DC.",
      "gold_sql": "SELECT Title FROM Album JOIN Artist ON Album.ArtistId = Artist.ArtistId WHERE Artist.Name = 'AC/DC'"
    },
    {
      "id": "Q12",
      "category": "join-2",
      "difficulty": "moderate",
      "question": "Which artists have the genre Jazz in their catalog?",
      "gold_sql": "SELECT DISTINCT Artist.Name FROM Artist JOIN Album ON Album.ArtistId = Artist.ArtistId JOIN Track ON Track.AlbumId = Album.AlbumId JOIN Genre ON Track.GenreId = Genre.GenreId WHERE Genre.Name = 'Jazz'"
    },
    {
      "id": "Q13",
      "category": "join-2",
      "difficulty": "moderate",
      "question": "List the names of tracks in the album 'Let There Be Rock'.",
      "gold_sql": "SELECT Track.Name FROM Track JOIN Album ON Track.AlbumId = Album.AlbumId WHERE Album.Title = 'Let There Be Rock'"
    },
    {
      "id": "Q14",
      "category": "group-by",
      "difficulty": "moderate",
      "question": "How many customers are in each country?",
      "gold_sql": "SELECT Country, COUNT(*) FROM Customer GROUP BY Country"
    },
    {
      "id": "Q15",
      "category": "group-by",
      "difficulty": "moderate",
      "question": "How many tracks does each genre have?",
      "gold_sql": "SELECT Genre.Name, COUNT(*) FROM Genre JOIN Track ON Track.GenreId = Genre.GenreId GROUP BY Genre.Name"
    },
    {
      "id": "Q16",
      "category": "group-by",
      "difficulty": "moderate",
      "question": "What is the total revenue per country?",
      "gold_sql": "SELECT BillingCountry, SUM(Total) FROM Invoice GROUP BY BillingCountry"
    },
    {
      "id": "Q17",
      "category": "top-n",
      "difficulty": "moderate",
      "question": "Who are the top 5 customers by total spending?",
      "gold_sql": "SELECT Customer.FirstName, Customer.LastName, SUM(Invoice.Total) AS total FROM Customer JOIN Invoice ON Customer.CustomerId = Invoice.CustomerId GROUP BY Customer.CustomerId ORDER BY total DESC LIMIT 5"
    },
    {
      "id": "Q18",
      "category": "top-n",
      "difficulty": "moderate",
      "question": "What are the 5 longest tracks?",
      "gold_sql": "SELECT Name FROM Track ORDER BY Milliseconds DESC LIMIT 5"
    },
    {
      "id": "Q19",
      "category": "top-n",
      "difficulty": "moderate",
      "question": "Which 3 countries have the most customers, and how many customers does each have?",
      "gold_sql": "SELECT Country, COUNT(*) AS n FROM Customer GROUP BY Country ORDER BY n DESC LIMIT 3"
    },
    {
      "id": "Q20",
      "category": "top-n",
      "difficulty": "moderate",
      "question": "Which artist has the most albums?",
      "gold_sql": "SELECT Artist.Name FROM Artist JOIN Album ON Album.ArtistId = Artist.ArtistId GROUP BY Artist.ArtistId ORDER BY COUNT(*) DESC LIMIT 1"
    },
    {
      "id": "Q21",
      "category": "date-filter",
      "difficulty": "moderate",
      "question": "How many invoices were issued in 2010?",
      "gold_sql": "SELECT COUNT(*) FROM Invoice WHERE strftime('%Y', InvoiceDate) = '2010'"
    },
    {
      "id": "Q22",
      "category": "date-filter",
      "difficulty": "moderate",
      "question": "What is the total revenue for the year 2011?",
      "gold_sql": "SELECT SUM(Total) FROM Invoice WHERE strftime('%Y', InvoiceDate) = '2011'"
    },
    {
      "id": "Q23",
      "category": "join-3",
      "difficulty": "moderate",
      "question": "Total revenue per genre.",
      "gold_sql": "SELECT Genre.Name, SUM(InvoiceLine.UnitPrice * InvoiceLine.Quantity) AS revenue FROM Genre JOIN Track ON Track.GenreId = Genre.GenreId JOIN InvoiceLine ON InvoiceLine.TrackId = Track.TrackId GROUP BY Genre.Name"
    },
    {
      "id": "Q24",
      "category": "join-3",
      "difficulty": "moderate",
      "question": "Which employee has the highest total sales?",
      "gold_sql": "SELECT Employee.FirstName, Employee.LastName FROM Employee JOIN Customer ON Customer.SupportRepId = Employee.EmployeeId JOIN Invoice ON Invoice.CustomerId = Customer.CustomerId GROUP BY Employee.EmployeeId ORDER BY SUM(Invoice.Total) DESC LIMIT 1"
    },
    {
      "id": "Q25",
      "category": "join-3",
      "difficulty": "moderate",
      "question": "List the distinct tracks in the playlist '90’s Music'.",
      "gold_sql": "SELECT DISTINCT Track.Name FROM Track JOIN PlaylistTrack ON PlaylistTrack.TrackId = Track.TrackId JOIN Playlist ON Playlist.PlaylistId = PlaylistTrack.PlaylistId WHERE Playlist.Name = '90’s Music'"
    },
    {
      "id": "Q26",
      "category": "aggregation",
      "difficulty": "moderate",
      "question": "What is the average invoice total per country?",
      "gold_sql": "SELECT BillingCountry, AVG(Total) FROM Invoice GROUP BY BillingCountry"
    },
    {
      "id": "Q27",
      "category": "having",
      "difficulty": "moderate",
      "question": "Which countries have more than 5 customers?",
      "gold_sql": "SELECT Country FROM Customer GROUP BY Country HAVING COUNT(*) > 5"
    },
    {
      "id": "Q28",
      "category": "having",
      "difficulty": "moderate",
      "question": "Which artists have more than 3 albums?",
      "gold_sql": "SELECT Artist.Name FROM Artist JOIN Album ON Album.ArtistId = Artist.ArtistId GROUP BY Artist.ArtistId HAVING COUNT(*) > 3"
    },
    {
      "id": "Q29",
      "category": "filter",
      "difficulty": "easy",
      "question": "Which tracks belong to the Rock genre?",
      "gold_sql": "SELECT Track.Name FROM Track JOIN Genre ON Track.GenreId = Genre.GenreId WHERE Genre.Name = 'Rock'"
    },
    {
      "id": "Q30",
      "category": "list",
      "difficulty": "easy",
      "question": "List all playlists.",
      "gold_sql": "SELECT Name FROM Playlist"
    },
    {
      "id": "H01",
      "category": "filter",
      "difficulty": "easy",
      "split": "held-out",
      "question": "Which employees are based in Calgary?",
      "gold_sql": "SELECT FirstName, LastName FROM Employee WHERE City = 'Calgary'"
    },
    {
      "id": "H02",
      "category": "filter",
      "difficulty": "easy",
      "split": "held-out",
      "question": "List the invoice IDs and totals for invoices billed to the city of Paris.",
      "gold_sql": "SELECT InvoiceId, Total FROM Invoice WHERE BillingCity = 'Paris'"
    },
    {
      "id": "H03",
      "category": "count",
      "difficulty": "easy",
      "split": "held-out",
      "question": "How many employees report to nobody (i.e., have no manager)?",
      "gold_sql": "SELECT COUNT(*) FROM Employee WHERE ReportsTo IS NULL"
    },
    {
      "id": "H04",
      "category": "aggregation",
      "difficulty": "easy",
      "split": "held-out",
      "question": "What is the minimum track price?",
      "gold_sql": "SELECT MIN(UnitPrice) FROM Track"
    },
    {
      "id": "H05",
      "category": "join-2",
      "difficulty": "moderate",
      "split": "held-out",
      "question": "List the genres of tracks on the album 'Restless and Wild'.",
      "gold_sql": "SELECT DISTINCT Genre.Name FROM Genre JOIN Track ON Track.GenreId = Genre.GenreId JOIN Album ON Track.AlbumId = Album.AlbumId WHERE Album.Title = 'Restless and Wild'"
    },
    {
      "id": "H06",
      "category": "join-2",
      "difficulty": "moderate",
      "split": "held-out",
      "question": "Which artist made the album 'Big Ones'?",
      "gold_sql": "SELECT Artist.Name FROM Artist JOIN Album ON Album.ArtistId = Artist.ArtistId WHERE Album.Title = 'Big Ones'"
    },
    {
      "id": "H07",
      "category": "top-n",
      "difficulty": "moderate",
      "split": "held-out",
      "question": "Which 3 genres have the most tracks?",
      "gold_sql": "SELECT Genre.Name FROM Genre JOIN Track ON Track.GenreId = Genre.GenreId GROUP BY Genre.GenreId ORDER BY COUNT(*) DESC LIMIT 3"
    },
    {
      "id": "H08",
      "category": "top-n",
      "difficulty": "moderate",
      "split": "held-out",
      "question": "Which customer made the largest single invoice?",
      "gold_sql": "SELECT Customer.FirstName, Customer.LastName FROM Customer JOIN Invoice ON Invoice.CustomerId = Customer.CustomerId ORDER BY Invoice.Total DESC LIMIT 1"
    },
    {
      "id": "H09",
      "category": "top-n",
      "difficulty": "moderate",
      "split": "held-out",
      "question": "Top 10 longest albums by total track duration.",
      "gold_sql": "SELECT Album.Title FROM Album JOIN Track ON Track.AlbumId = Album.AlbumId GROUP BY Album.AlbumId ORDER BY SUM(Track.Milliseconds) DESC LIMIT 10"
    },
    {
      "id": "H10",
      "category": "group-by",
      "difficulty": "moderate",
      "split": "held-out",
      "question": "For each customer, list their first name, last name, and how many invoices they have.",
      "gold_sql": "SELECT Customer.FirstName, Customer.LastName, COUNT(Invoice.InvoiceId) FROM Customer JOIN Invoice ON Customer.CustomerId = Invoice.CustomerId GROUP BY Customer.CustomerId"
    },
    {
      "id": "H11",
      "category": "date-filter",
      "difficulty": "moderate",
      "split": "held-out",
      "question": "Total revenue for January 2010.",
      "gold_sql": "SELECT SUM(Total) FROM Invoice WHERE strftime('%Y-%m', InvoiceDate) = '2010-01'"
    },
    {
      "id": "H12",
      "category": "date-filter",
      "difficulty": "moderate",
      "split": "held-out",
      "question": "How many invoices were issued in the second quarter of 2012?",
      "gold_sql": "SELECT COUNT(*) FROM Invoice WHERE strftime('%Y-%m', InvoiceDate) BETWEEN '2012-04' AND '2012-06'"
    },
    {
      "id": "H13",
      "category": "having",
      "difficulty": "moderate",
      "split": "held-out",
      "question": "Which genres have at least 100 tracks?",
      "gold_sql": "SELECT Genre.Name FROM Genre JOIN Track ON Track.GenreId = Genre.GenreId GROUP BY Genre.GenreId HAVING COUNT(*) >= 100"
    },
    {
      "id": "H14",
      "category": "having",
      "difficulty": "moderate",
      "split": "held-out",
      "question": "Which customers have spent more than 40 dollars in total?",
      "gold_sql": "SELECT Customer.FirstName, Customer.LastName FROM Customer JOIN Invoice ON Invoice.CustomerId = Customer.CustomerId GROUP BY Customer.CustomerId HAVING SUM(Invoice.Total) > 40"
    },
    {
      "id": "H15",
      "category": "join-3",
      "difficulty": "moderate",
      "split": "held-out",
      "question": "What is the total revenue for the country Germany?",
      "gold_sql": "SELECT SUM(Total) FROM Invoice WHERE BillingCountry = 'Germany'"
    },
    {
      "id": "H16",
      "category": "join-3",
      "difficulty": "moderate",
      "split": "held-out",
      "question": "Which media types are used by tracks in the Pop genre?",
      "gold_sql": "SELECT DISTINCT MediaType.Name FROM MediaType JOIN Track ON Track.MediaTypeId = MediaType.MediaTypeId JOIN Genre ON Track.GenreId = Genre.GenreId WHERE Genre.Name = 'Pop'"
    },
    {
      "id": "H17",
      "category": "join-3",
      "difficulty": "moderate",
      "split": "held-out",
      "question": "Which customers bought tracks from the album 'Greatest Hits'?",
      "gold_sql": "SELECT DISTINCT Customer.FirstName, Customer.LastName FROM Customer JOIN Invoice ON Invoice.CustomerId = Customer.CustomerId JOIN InvoiceLine ON InvoiceLine.InvoiceId = Invoice.InvoiceId JOIN Track ON Track.TrackId = InvoiceLine.TrackId JOIN Album ON Track.AlbumId = Album.AlbumId WHERE Album.Title = 'Greatest Hits'"
    },
    {
      "id": "H18",
      "category": "filter",
      "difficulty": "moderate",
      "split": "held-out",
      "question": "List the tracks that cost more than 1 dollar.",
      "gold_sql": "SELECT Name FROM Track WHERE UnitPrice > 1"
    },
    {
      "id": "H19",
      "category": "aggregation",
      "difficulty": "moderate",
      "split": "held-out",
      "question": "What fraction of tracks belong to the Rock genre?",
      "gold_sql": "SELECT CAST(SUM(CASE WHEN Genre.Name = 'Rock' THEN 1 ELSE 0 END) AS REAL) / COUNT(*) FROM Track JOIN Genre ON Track.GenreId = Genre.GenreId"
    },
    {
      "id": "H20",
      "category": "group-by",
      "difficulty": "moderate",
      "split": "held-out",
      "question": "Average track length per genre.",
      "gold_sql": "SELECT Genre.Name, AVG(Track.Milliseconds) FROM Genre JOIN Track ON Track.GenreId = Genre.GenreId GROUP BY Genre.Name"
    },
    {
      "id": "H21",
      "category": "top-n",
      "difficulty": "moderate",
      "split": "held-out",
      "question": "Top 5 cities by total revenue.",
      "gold_sql": "SELECT BillingCity FROM Invoice GROUP BY BillingCity ORDER BY SUM(Total) DESC LIMIT 5"
    },
    {
      "id": "H22",
      "category": "list",
      "difficulty": "easy",
      "split": "held-out",
      "question": "What are the distinct billing countries from invoices?",
      "gold_sql": "SELECT DISTINCT BillingCountry FROM Invoice"
    },
    {
      "id": "H23",
      "category": "filter",
      "difficulty": "moderate",
      "split": "held-out",
      "question": "List the albums that have zero tracks linked to them.",
      "gold_sql": "SELECT Title FROM Album WHERE AlbumId NOT IN (SELECT AlbumId FROM Track WHERE AlbumId IS NOT NULL)"
    },
    {
      "id": "H24",
      "category": "filter",
      "difficulty": "moderate",
      "split": "held-out",
      "question": "Which customers have never made a purchase?",
      "gold_sql": "SELECT FirstName, LastName FROM Customer WHERE CustomerId NOT IN (SELECT CustomerId FROM Invoice)"
    },
    {
      "id": "H25",
      "category": "join-2",
      "difficulty": "moderate",
      "split": "held-out",
      "question": "Names of tracks composed by Angus Young.",
      "gold_sql": "SELECT Name FROM Track WHERE Composer LIKE '%Angus Young%'"
    },
    {
      "id": "H26",
      "category": "aggregation",
      "difficulty": "moderate",
      "split": "held-out",
      "question": "How many distinct genres are represented in the catalog?",
      "gold_sql": "SELECT COUNT(DISTINCT GenreId) FROM Track"
    },
    {
      "id": "H27",
      "category": "join-2",
      "difficulty": "easy",
      "split": "held-out",
      "question": "Which playlists contain the track 'Eruption'?",
      "gold_sql": "SELECT DISTINCT Playlist.Name FROM Playlist JOIN PlaylistTrack ON PlaylistTrack.PlaylistId = Playlist.PlaylistId JOIN Track ON Track.TrackId = PlaylistTrack.TrackId WHERE Track.Name = 'Eruption'"
    },
    {
      "id": "H28",
      "category": "top-n",
      "difficulty": "moderate",
      "split": "held-out",
      "question": "Which is the cheapest track in the catalog?",
      "gold_sql": "SELECT Name FROM Track ORDER BY UnitPrice ASC, TrackId ASC LIMIT 1"
    },
    {
      "id": "H29",
      "category": "count",
      "difficulty": "easy",
      "split": "held-out",
      "question": "How many tracks does the album 'Use Your Illusion I' have?",
      "gold_sql": "SELECT COUNT(*) FROM Track JOIN Album ON Track.AlbumId = Album.AlbumId WHERE Album.Title = 'Use Your Illusion I'"
    },
    {
      "id": "H30",
      "category": "group-by",
      "difficulty": "moderate",
      "split": "held-out",
      "question": "Number of invoices per year.",
      "gold_sql": "SELECT strftime('%Y', InvoiceDate) AS year, COUNT(*) FROM Invoice GROUP BY year"
    }
  ]
}