| { |
| "name": "Chinook demo benchmark", |
| "description": "Curated NL→SQL questions on the Chinook music store schema. Designed as a realistic 'business analyst workload' rather than the hard-mode BIRD edge cases. Target accuracy ≥ 90% — these are the kinds of questions a product would actually serve.", |
| "db_id": "chinook", |
| "dialect": "sqlite", |
| "questions": [ |
| { |
| "id": "Q01", |
| "category": "count", |
| "difficulty": "easy", |
| "question": "How many albums are in the catalog?", |
| "gold_sql": "SELECT COUNT(*) FROM Album" |
| }, |
| { |
| "id": "Q02", |
| "category": "count", |
| "difficulty": "easy", |
| "question": "How many tracks are there in total?", |
| "gold_sql": "SELECT COUNT(*) FROM Track" |
| }, |
| { |
| "id": "Q03", |
| "category": "count", |
| "difficulty": "easy", |
| "question": "How many customers do we have?", |
| "gold_sql": "SELECT COUNT(*) FROM Customer" |
| }, |
| { |
| "id": "Q04", |
| "category": "list", |
| "difficulty": "easy", |
| "question": "List the names of all genres.", |
| "gold_sql": "SELECT Name FROM Genre" |
| }, |
| { |
| "id": "Q05", |
| "category": "list", |
| "difficulty": "easy", |
| "question": "List all media types.", |
| "gold_sql": "SELECT Name FROM MediaType" |
| }, |
| { |
| "id": "Q06", |
| "category": "filter", |
| "difficulty": "easy", |
| "question": "Which customers are from Germany?", |
| "gold_sql": "SELECT FirstName, LastName FROM Customer WHERE Country = 'Germany'" |
| }, |
| { |
| "id": "Q07", |
| "category": "filter", |
| "difficulty": "easy", |
| "question": "List the customers from Brazil.", |
| "gold_sql": "SELECT FirstName, LastName FROM Customer WHERE Country = 'Brazil'" |
| }, |
| { |
| "id": "Q08", |
| "category": "aggregation", |
| "difficulty": "easy", |
| "question": "What is the average track length in milliseconds?", |
| "gold_sql": "SELECT AVG(Milliseconds) FROM Track" |
| }, |
| { |
| "id": "Q09", |
| "category": "aggregation", |
| "difficulty": "easy", |
| "question": "What is the total revenue across all invoices?", |
| "gold_sql": "SELECT SUM(Total) FROM Invoice" |
| }, |
| { |
| "id": "Q10", |
| "category": "aggregation", |
| "difficulty": "easy", |
| "question": "What is the most expensive track price?", |
| "gold_sql": "SELECT MAX(UnitPrice) FROM Track" |
| }, |
| { |
| "id": "Q11", |
| "category": "join-2", |
| "difficulty": "moderate", |
| "question": "List all album titles by the artist AC/DC.", |
| "gold_sql": "SELECT Title FROM Album JOIN Artist ON Album.ArtistId = Artist.ArtistId WHERE Artist.Name = 'AC/DC'" |
| }, |
| { |
| "id": "Q12", |
| "category": "join-2", |
| "difficulty": "moderate", |
| "question": "Which artists have the genre Jazz in their catalog?", |
| "gold_sql": "SELECT DISTINCT Artist.Name FROM Artist JOIN Album ON Album.ArtistId = Artist.ArtistId JOIN Track ON Track.AlbumId = Album.AlbumId JOIN Genre ON Track.GenreId = Genre.GenreId WHERE Genre.Name = 'Jazz'" |
| }, |
| { |
| "id": "Q13", |
| "category": "join-2", |
| "difficulty": "moderate", |
| "question": "List the names of tracks in the album 'Let There Be Rock'.", |
| "gold_sql": "SELECT Track.Name FROM Track JOIN Album ON Track.AlbumId = Album.AlbumId WHERE Album.Title = 'Let There Be Rock'" |
| }, |
| { |
| "id": "Q14", |
| "category": "group-by", |
| "difficulty": "moderate", |
| "question": "How many customers are in each country?", |
| "gold_sql": "SELECT Country, COUNT(*) FROM Customer GROUP BY Country" |
| }, |
| { |
| "id": "Q15", |
| "category": "group-by", |
| "difficulty": "moderate", |
| "question": "How many tracks does each genre have?", |
| "gold_sql": "SELECT Genre.Name, COUNT(*) FROM Genre JOIN Track ON Track.GenreId = Genre.GenreId GROUP BY Genre.Name" |
| }, |
| { |
| "id": "Q16", |
| "category": "group-by", |
| "difficulty": "moderate", |
| "question": "What is the total revenue per country?", |
| "gold_sql": "SELECT BillingCountry, SUM(Total) FROM Invoice GROUP BY BillingCountry" |
| }, |
| { |
| "id": "Q17", |
| "category": "top-n", |
| "difficulty": "moderate", |
| "question": "Who are the top 5 customers by total spending?", |
| "gold_sql": "SELECT Customer.FirstName, Customer.LastName, SUM(Invoice.Total) AS total FROM Customer JOIN Invoice ON Customer.CustomerId = Invoice.CustomerId GROUP BY Customer.CustomerId ORDER BY total DESC LIMIT 5" |
| }, |
| { |
| "id": "Q18", |
| "category": "top-n", |
| "difficulty": "moderate", |
| "question": "What are the 5 longest tracks?", |
| "gold_sql": "SELECT Name FROM Track ORDER BY Milliseconds DESC LIMIT 5" |
| }, |
| { |
| "id": "Q19", |
| "category": "top-n", |
| "difficulty": "moderate", |
| "question": "Which 3 countries have the most customers, and how many customers does each have?", |
| "gold_sql": "SELECT Country, COUNT(*) AS n FROM Customer GROUP BY Country ORDER BY n DESC LIMIT 3" |
| }, |
| { |
| "id": "Q20", |
| "category": "top-n", |
| "difficulty": "moderate", |
| "question": "Which artist has the most albums?", |
| "gold_sql": "SELECT Artist.Name FROM Artist JOIN Album ON Album.ArtistId = Artist.ArtistId GROUP BY Artist.ArtistId ORDER BY COUNT(*) DESC LIMIT 1" |
| }, |
| { |
| "id": "Q21", |
| "category": "date-filter", |
| "difficulty": "moderate", |
| "question": "How many invoices were issued in 2010?", |
| "gold_sql": "SELECT COUNT(*) FROM Invoice WHERE strftime('%Y', InvoiceDate) = '2010'" |
| }, |
| { |
| "id": "Q22", |
| "category": "date-filter", |
| "difficulty": "moderate", |
| "question": "What is the total revenue for the year 2011?", |
| "gold_sql": "SELECT SUM(Total) FROM Invoice WHERE strftime('%Y', InvoiceDate) = '2011'" |
| }, |
| { |
| "id": "Q23", |
| "category": "join-3", |
| "difficulty": "moderate", |
| "question": "Total revenue per genre.", |
| "gold_sql": "SELECT Genre.Name, SUM(InvoiceLine.UnitPrice * InvoiceLine.Quantity) AS revenue FROM Genre JOIN Track ON Track.GenreId = Genre.GenreId JOIN InvoiceLine ON InvoiceLine.TrackId = Track.TrackId GROUP BY Genre.Name" |
| }, |
| { |
| "id": "Q24", |
| "category": "join-3", |
| "difficulty": "moderate", |
| "question": "Which employee has the highest total sales?", |
| "gold_sql": "SELECT Employee.FirstName, Employee.LastName FROM Employee JOIN Customer ON Customer.SupportRepId = Employee.EmployeeId JOIN Invoice ON Invoice.CustomerId = Customer.CustomerId GROUP BY Employee.EmployeeId ORDER BY SUM(Invoice.Total) DESC LIMIT 1" |
| }, |
| { |
| "id": "Q25", |
| "category": "join-3", |
| "difficulty": "moderate", |
| "question": "List the distinct tracks in the playlist '90’s Music'.", |
| "gold_sql": "SELECT DISTINCT Track.Name FROM Track JOIN PlaylistTrack ON PlaylistTrack.TrackId = Track.TrackId JOIN Playlist ON Playlist.PlaylistId = PlaylistTrack.PlaylistId WHERE Playlist.Name = '90’s Music'" |
| }, |
| { |
| "id": "Q26", |
| "category": "aggregation", |
| "difficulty": "moderate", |
| "question": "What is the average invoice total per country?", |
| "gold_sql": "SELECT BillingCountry, AVG(Total) FROM Invoice GROUP BY BillingCountry" |
| }, |
| { |
| "id": "Q27", |
| "category": "having", |
| "difficulty": "moderate", |
| "question": "Which countries have more than 5 customers?", |
| "gold_sql": "SELECT Country FROM Customer GROUP BY Country HAVING COUNT(*) > 5" |
| }, |
| { |
| "id": "Q28", |
| "category": "having", |
| "difficulty": "moderate", |
| "question": "Which artists have more than 3 albums?", |
| "gold_sql": "SELECT Artist.Name FROM Artist JOIN Album ON Album.ArtistId = Artist.ArtistId GROUP BY Artist.ArtistId HAVING COUNT(*) > 3" |
| }, |
| { |
| "id": "Q29", |
| "category": "filter", |
| "difficulty": "easy", |
| "question": "Which tracks belong to the Rock genre?", |
| "gold_sql": "SELECT Track.Name FROM Track JOIN Genre ON Track.GenreId = Genre.GenreId WHERE Genre.Name = 'Rock'" |
| }, |
| { |
| "id": "Q30", |
| "category": "list", |
| "difficulty": "easy", |
| "question": "List all playlists.", |
| "gold_sql": "SELECT Name FROM Playlist" |
| }, |
| { |
| "id": "H01", |
| "category": "filter", |
| "difficulty": "easy", |
| "split": "held-out", |
| "question": "Which employees are based in Calgary?", |
| "gold_sql": "SELECT FirstName, LastName FROM Employee WHERE City = 'Calgary'" |
| }, |
| { |
| "id": "H02", |
| "category": "filter", |
| "difficulty": "easy", |
| "split": "held-out", |
| "question": "List the invoice IDs and totals for invoices billed to the city of Paris.", |
| "gold_sql": "SELECT InvoiceId, Total FROM Invoice WHERE BillingCity = 'Paris'" |
| }, |
| { |
| "id": "H03", |
| "category": "count", |
| "difficulty": "easy", |
| "split": "held-out", |
| "question": "How many employees report to nobody (i.e., have no manager)?", |
| "gold_sql": "SELECT COUNT(*) FROM Employee WHERE ReportsTo IS NULL" |
| }, |
| { |
| "id": "H04", |
| "category": "aggregation", |
| "difficulty": "easy", |
| "split": "held-out", |
| "question": "What is the minimum track price?", |
| "gold_sql": "SELECT MIN(UnitPrice) FROM Track" |
| }, |
| { |
| "id": "H05", |
| "category": "join-2", |
| "difficulty": "moderate", |
| "split": "held-out", |
| "question": "List the genres of tracks on the album 'Restless and Wild'.", |
| "gold_sql": "SELECT DISTINCT Genre.Name FROM Genre JOIN Track ON Track.GenreId = Genre.GenreId JOIN Album ON Track.AlbumId = Album.AlbumId WHERE Album.Title = 'Restless and Wild'" |
| }, |
| { |
| "id": "H06", |
| "category": "join-2", |
| "difficulty": "moderate", |
| "split": "held-out", |
| "question": "Which artist made the album 'Big Ones'?", |
| "gold_sql": "SELECT Artist.Name FROM Artist JOIN Album ON Album.ArtistId = Artist.ArtistId WHERE Album.Title = 'Big Ones'" |
| }, |
| { |
| "id": "H07", |
| "category": "top-n", |
| "difficulty": "moderate", |
| "split": "held-out", |
| "question": "Which 3 genres have the most tracks?", |
| "gold_sql": "SELECT Genre.Name FROM Genre JOIN Track ON Track.GenreId = Genre.GenreId GROUP BY Genre.GenreId ORDER BY COUNT(*) DESC LIMIT 3" |
| }, |
| { |
| "id": "H08", |
| "category": "top-n", |
| "difficulty": "moderate", |
| "split": "held-out", |
| "question": "Which customer made the largest single invoice?", |
| "gold_sql": "SELECT Customer.FirstName, Customer.LastName FROM Customer JOIN Invoice ON Invoice.CustomerId = Customer.CustomerId ORDER BY Invoice.Total DESC LIMIT 1" |
| }, |
| { |
| "id": "H09", |
| "category": "top-n", |
| "difficulty": "moderate", |
| "split": "held-out", |
| "question": "Top 10 longest albums by total track duration.", |
| "gold_sql": "SELECT Album.Title FROM Album JOIN Track ON Track.AlbumId = Album.AlbumId GROUP BY Album.AlbumId ORDER BY SUM(Track.Milliseconds) DESC LIMIT 10" |
| }, |
| { |
| "id": "H10", |
| "category": "group-by", |
| "difficulty": "moderate", |
| "split": "held-out", |
| "question": "For each customer, list their first name, last name, and how many invoices they have.", |
| "gold_sql": "SELECT Customer.FirstName, Customer.LastName, COUNT(Invoice.InvoiceId) FROM Customer JOIN Invoice ON Customer.CustomerId = Invoice.CustomerId GROUP BY Customer.CustomerId" |
| }, |
| { |
| "id": "H11", |
| "category": "date-filter", |
| "difficulty": "moderate", |
| "split": "held-out", |
| "question": "Total revenue for January 2010.", |
| "gold_sql": "SELECT SUM(Total) FROM Invoice WHERE strftime('%Y-%m', InvoiceDate) = '2010-01'" |
| }, |
| { |
| "id": "H12", |
| "category": "date-filter", |
| "difficulty": "moderate", |
| "split": "held-out", |
| "question": "How many invoices were issued in the second quarter of 2012?", |
| "gold_sql": "SELECT COUNT(*) FROM Invoice WHERE strftime('%Y-%m', InvoiceDate) BETWEEN '2012-04' AND '2012-06'" |
| }, |
| { |
| "id": "H13", |
| "category": "having", |
| "difficulty": "moderate", |
| "split": "held-out", |
| "question": "Which genres have at least 100 tracks?", |
| "gold_sql": "SELECT Genre.Name FROM Genre JOIN Track ON Track.GenreId = Genre.GenreId GROUP BY Genre.GenreId HAVING COUNT(*) >= 100" |
| }, |
| { |
| "id": "H14", |
| "category": "having", |
| "difficulty": "moderate", |
| "split": "held-out", |
| "question": "Which customers have spent more than 40 dollars in total?", |
| "gold_sql": "SELECT Customer.FirstName, Customer.LastName FROM Customer JOIN Invoice ON Invoice.CustomerId = Customer.CustomerId GROUP BY Customer.CustomerId HAVING SUM(Invoice.Total) > 40" |
| }, |
| { |
| "id": "H15", |
| "category": "join-3", |
| "difficulty": "moderate", |
| "split": "held-out", |
| "question": "What is the total revenue for the country Germany?", |
| "gold_sql": "SELECT SUM(Total) FROM Invoice WHERE BillingCountry = 'Germany'" |
| }, |
| { |
| "id": "H16", |
| "category": "join-3", |
| "difficulty": "moderate", |
| "split": "held-out", |
| "question": "Which media types are used by tracks in the Pop genre?", |
| "gold_sql": "SELECT DISTINCT MediaType.Name FROM MediaType JOIN Track ON Track.MediaTypeId = MediaType.MediaTypeId JOIN Genre ON Track.GenreId = Genre.GenreId WHERE Genre.Name = 'Pop'" |
| }, |
| { |
| "id": "H17", |
| "category": "join-3", |
| "difficulty": "moderate", |
| "split": "held-out", |
| "question": "Which customers bought tracks from the album 'Greatest Hits'?", |
| "gold_sql": "SELECT DISTINCT Customer.FirstName, Customer.LastName FROM Customer JOIN Invoice ON Invoice.CustomerId = Customer.CustomerId JOIN InvoiceLine ON InvoiceLine.InvoiceId = Invoice.InvoiceId JOIN Track ON Track.TrackId = InvoiceLine.TrackId JOIN Album ON Track.AlbumId = Album.AlbumId WHERE Album.Title = 'Greatest Hits'" |
| }, |
| { |
| "id": "H18", |
| "category": "filter", |
| "difficulty": "moderate", |
| "split": "held-out", |
| "question": "List the tracks that cost more than 1 dollar.", |
| "gold_sql": "SELECT Name FROM Track WHERE UnitPrice > 1" |
| }, |
| { |
| "id": "H19", |
| "category": "aggregation", |
| "difficulty": "moderate", |
| "split": "held-out", |
| "question": "What fraction of tracks belong to the Rock genre?", |
| "gold_sql": "SELECT CAST(SUM(CASE WHEN Genre.Name = 'Rock' THEN 1 ELSE 0 END) AS REAL) / COUNT(*) FROM Track JOIN Genre ON Track.GenreId = Genre.GenreId" |
| }, |
| { |
| "id": "H20", |
| "category": "group-by", |
| "difficulty": "moderate", |
| "split": "held-out", |
| "question": "Average track length per genre.", |
| "gold_sql": "SELECT Genre.Name, AVG(Track.Milliseconds) FROM Genre JOIN Track ON Track.GenreId = Genre.GenreId GROUP BY Genre.Name" |
| }, |
| { |
| "id": "H21", |
| "category": "top-n", |
| "difficulty": "moderate", |
| "split": "held-out", |
| "question": "Top 5 cities by total revenue.", |
| "gold_sql": "SELECT BillingCity FROM Invoice GROUP BY BillingCity ORDER BY SUM(Total) DESC LIMIT 5" |
| }, |
| { |
| "id": "H22", |
| "category": "list", |
| "difficulty": "easy", |
| "split": "held-out", |
| "question": "What are the distinct billing countries from invoices?", |
| "gold_sql": "SELECT DISTINCT BillingCountry FROM Invoice" |
| }, |
| { |
| "id": "H23", |
| "category": "filter", |
| "difficulty": "moderate", |
| "split": "held-out", |
| "question": "List the albums that have zero tracks linked to them.", |
| "gold_sql": "SELECT Title FROM Album WHERE AlbumId NOT IN (SELECT AlbumId FROM Track WHERE AlbumId IS NOT NULL)" |
| }, |
| { |
| "id": "H24", |
| "category": "filter", |
| "difficulty": "moderate", |
| "split": "held-out", |
| "question": "Which customers have never made a purchase?", |
| "gold_sql": "SELECT FirstName, LastName FROM Customer WHERE CustomerId NOT IN (SELECT CustomerId FROM Invoice)" |
| }, |
| { |
| "id": "H25", |
| "category": "join-2", |
| "difficulty": "moderate", |
| "split": "held-out", |
| "question": "Names of tracks composed by Angus Young.", |
| "gold_sql": "SELECT Name FROM Track WHERE Composer LIKE '%Angus Young%'" |
| }, |
| { |
| "id": "H26", |
| "category": "aggregation", |
| "difficulty": "moderate", |
| "split": "held-out", |
| "question": "How many distinct genres are represented in the catalog?", |
| "gold_sql": "SELECT COUNT(DISTINCT GenreId) FROM Track" |
| }, |
| { |
| "id": "H27", |
| "category": "join-2", |
| "difficulty": "easy", |
| "split": "held-out", |
| "question": "Which playlists contain the track 'Eruption'?", |
| "gold_sql": "SELECT DISTINCT Playlist.Name FROM Playlist JOIN PlaylistTrack ON PlaylistTrack.PlaylistId = Playlist.PlaylistId JOIN Track ON Track.TrackId = PlaylistTrack.TrackId WHERE Track.Name = 'Eruption'" |
| }, |
| { |
| "id": "H28", |
| "category": "top-n", |
| "difficulty": "moderate", |
| "split": "held-out", |
| "question": "Which is the cheapest track in the catalog?", |
| "gold_sql": "SELECT Name FROM Track ORDER BY UnitPrice ASC, TrackId ASC LIMIT 1" |
| }, |
| { |
| "id": "H29", |
| "category": "count", |
| "difficulty": "easy", |
| "split": "held-out", |
| "question": "How many tracks does the album 'Use Your Illusion I' have?", |
| "gold_sql": "SELECT COUNT(*) FROM Track JOIN Album ON Track.AlbumId = Album.AlbumId WHERE Album.Title = 'Use Your Illusion I'" |
| }, |
| { |
| "id": "H30", |
| "category": "group-by", |
| "difficulty": "moderate", |
| "split": "held-out", |
| "question": "Number of invoices per year.", |
| "gold_sql": "SELECT strftime('%Y', InvoiceDate) AS year, COUNT(*) FROM Invoice GROUP BY year" |
| } |
| ] |
| } |
|
|
