backend/evaluation/datasets/test.json

[
    {"id":"test_001","question":"Show all strategic segment accounts","expected_sql":"SELECT account_name, industry, region, health_score, churn_risk FROM accounts WHERE segment = 'strategic' ORDER BY health_score DESC","difficulty":"easy","category":"data_query","expected_tables":["accounts"]},
    {"id":"test_002","question":"How many active subscriptions are there?","expected_sql":"SELECT COUNT(*) AS active_subscriptions FROM subscriptions WHERE status = 'active'","difficulty":"easy","category":"aggregation","expected_tables":["subscriptions"]},
    {"id":"test_003","question":"List all P2 tickets that are pending customer response","expected_sql":"SELECT ticket_id, subject, category, created_at FROM support_tickets WHERE priority = 'P2' AND status = 'pending_customer' ORDER BY created_at DESC","difficulty":"easy","category":"data_query","expected_tables":["support_tickets"]},
    {"id":"test_004","question":"Show inactive employees","expected_sql":"SELECT full_name, role, location, hire_date FROM employees WHERE active = FALSE","difficulty":"easy","category":"data_query","expected_tables":["employees"]},
    {"id":"test_005","question":"How many uncollectible invoices exist?","expected_sql":"SELECT COUNT(*) AS uncollectible, SUM(total) AS total_value FROM invoices WHERE status = 'uncollectible'","difficulty":"easy","category":"aggregation","expected_tables":["invoices"]},
    {"id":"test_006","question":"List all mid_market accounts by region","expected_sql":"SELECT account_name, industry, region, country, health_score FROM accounts WHERE segment = 'mid_market' ORDER BY region, account_name","difficulty":"easy","category":"data_query","expected_tables":["accounts"]},
    {"id":"test_007","question":"Show department budgets by region","expected_sql":"SELECT region, SUM(annual_budget) AS total_budget, COUNT(*) AS dept_count FROM departments GROUP BY region ORDER BY total_budget DESC","difficulty":"easy","category":"aggregation","expected_tables":["departments"]},
    {"id":"test_008","question":"How many workspace users are service accounts?","expected_sql":"SELECT COUNT(*) AS service_accounts FROM workspace_users WHERE is_service_account = TRUE","difficulty":"easy","category":"aggregation","expected_tables":["workspace_users"]},
    {"id":"test_009","question":"Show all refunded payments","expected_sql":"SELECT payment_id, account_id, amount, paid_at FROM payments WHERE payment_status = 'refunded' ORDER BY amount DESC","difficulty":"easy","category":"data_query","expected_tables":["payments"]},
    {"id":"test_010","question":"How many accounts are in each industry?","expected_sql":"SELECT industry, COUNT(*) AS account_count FROM accounts GROUP BY industry ORDER BY account_count DESC","difficulty":"easy","category":"aggregation","expected_tables":["accounts"]},
    {"id":"test_011","question":"Show subscriptions on the Enterprise plan with their ARR","expected_sql":"SELECT a.account_name, s.contracted_arr, s.seats_purchased, s.status FROM subscriptions s JOIN accounts a ON s.account_id = a.account_id JOIN plans p ON s.plan_id = p.plan_id WHERE p.plan_name = 'Enterprise' ORDER BY s.contracted_arr DESC","difficulty":"medium","category":"data_query","expected_tables":["subscriptions","accounts","plans"]},
    {"id":"test_012","question":"What is the average ticket resolution time by category?","expected_sql":"SELECT category, AVG(resolution_minutes) AS avg_resolution_min, COUNT(*) AS tickets FROM support_tickets WHERE resolution_minutes IS NOT NULL GROUP BY category ORDER BY avg_resolution_min ASC","difficulty":"medium","category":"aggregation","expected_tables":["support_tickets"]},
    {"id":"test_013","question":"Show monthly query volume from the audit log","expected_sql":"SELECT DATE_FORMAT(created_at, '%Y-%m') AS month, COUNT(*) AS query_count, SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) AS successful FROM query_audit_log GROUP BY month ORDER BY month","difficulty":"medium","category":"aggregation","expected_tables":["query_audit_log"]},
    {"id":"test_014","question":"Which cloud regions have the most workspaces?","expected_sql":"SELECT cloud_region, COUNT(*) AS workspace_count FROM workspaces GROUP BY cloud_region ORDER BY workspace_count DESC","difficulty":"medium","category":"aggregation","expected_tables":["workspaces"]},
    {"id":"test_015","question":"Show opportunity pipeline by source channel","expected_sql":"SELECT source, COUNT(*) AS deals, SUM(expected_arr) AS total_arr, AVG(probability_pct) AS avg_prob FROM opportunities WHERE stage NOT IN ('closed_won','closed_lost') GROUP BY source ORDER BY total_arr DESC","difficulty":"medium","category":"aggregation","expected_tables":["opportunities"]},
    {"id":"test_016","question":"List accounts with trialing subscriptions","expected_sql":"SELECT a.account_name, a.segment, s.contracted_arr, p.plan_name, s.start_date FROM subscriptions s JOIN accounts a ON s.account_id = a.account_id JOIN plans p ON s.plan_id = p.plan_id WHERE s.status = 'trialing' ORDER BY s.start_date DESC","difficulty":"medium","category":"data_query","expected_tables":["subscriptions","accounts","plans"]},
    {"id":"test_017","question":"What is the total compute usage by account segment?","expected_sql":"SELECT a.segment, SUM(pud.compute_seconds) AS total_compute_sec, AVG(pud.compute_seconds) AS avg_compute_sec FROM product_usage_daily pud JOIN accounts a ON pud.account_id = a.account_id GROUP BY a.segment ORDER BY total_compute_sec DESC","difficulty":"medium","category":"aggregation","expected_tables":["product_usage_daily","accounts"]},
    {"id":"test_018","question":"Show the number of contacts per account for enterprise customers","expected_sql":"SELECT a.account_name, COUNT(c.contact_id) AS contact_count FROM contacts c JOIN accounts a ON c.account_id = a.account_id WHERE a.segment = 'enterprise' GROUP BY a.account_id, a.account_name ORDER BY contact_count DESC","difficulty":"medium","category":"aggregation","expected_tables":["contacts","accounts"]},
    {"id":"test_019","question":"What is the average p95 latency by workspace environment?","expected_sql":"SELECT w.environment, AVG(pud.p95_latency_ms) AS avg_p95_latency FROM product_usage_daily pud JOIN workspaces w ON pud.workspace_id = w.workspace_id GROUP BY w.environment ORDER BY avg_p95_latency DESC","difficulty":"medium","category":"aggregation","expected_tables":["product_usage_daily","workspaces"]},
    {"id":"test_020","question":"Show accounts by ARR band with average health score","expected_sql":"SELECT arr_band, COUNT(*) AS accounts, AVG(health_score) AS avg_health FROM accounts GROUP BY arr_band ORDER BY FIELD(arr_band,'under_25k','25k_100k','100k_500k','500k_plus')","difficulty":"medium","category":"aggregation","expected_tables":["accounts"]},
    {"id":"test_021","question":"Which employees have the highest quota attainment?","expected_sql":"SELECT e.full_name, e.role, e.quota_arr, SUM(o.expected_arr) AS closed_arr, ROUND(SUM(o.expected_arr) * 100.0 / e.quota_arr, 2) AS attainment_pct FROM employees e JOIN opportunities o ON e.employee_id = o.owner_employee_id WHERE o.stage = 'closed_won' AND e.quota_arr > 0 GROUP BY e.employee_id, e.full_name, e.role, e.quota_arr ORDER BY attainment_pct DESC LIMIT 10","difficulty":"hard","category":"aggregation","expected_tables":["employees","opportunities"]},
    {"id":"test_022","question":"Show the correlation between account health score and ticket volume","expected_sql":"SELECT a.account_name, a.health_score, a.churn_risk, COUNT(st.ticket_id) AS ticket_count FROM accounts a LEFT JOIN support_tickets st ON a.account_id = st.account_id GROUP BY a.account_id, a.account_name, a.health_score, a.churn_risk ORDER BY ticket_count DESC LIMIT 20","difficulty":"hard","category":"aggregation","expected_tables":["accounts","support_tickets"]},
    {"id":"test_023","question":"Calculate closed-won conversion rate by opportunity source","expected_sql":"SELECT source, COUNT(*) AS total_deals, SUM(CASE WHEN stage = 'closed_won' THEN 1 ELSE 0 END) AS won, ROUND(SUM(CASE WHEN stage = 'closed_won' THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 2) AS win_rate FROM opportunities GROUP BY source ORDER BY win_rate DESC","difficulty":"hard","category":"aggregation","expected_tables":["opportunities"]},
    {"id":"test_024","question":"Find accounts with cancelled subscriptions and active P1 tickets","expected_sql":"SELECT DISTINCT a.account_name, a.segment, s.contracted_arr, st.subject FROM accounts a JOIN subscriptions s ON a.account_id = s.account_id JOIN support_tickets st ON a.account_id = st.account_id WHERE s.status = 'cancelled' AND st.priority = 'P1' AND st.status IN ('open','pending_customer') ORDER BY s.contracted_arr DESC","difficulty":"hard","category":"data_query","expected_tables":["accounts","subscriptions","support_tickets"]},
    {"id":"test_025","question":"Show the top 5 features by failed query count","expected_sql":"SELECT fc.feature_name, p.product_name, SUM(pud.failed_query_count) AS total_failures, SUM(pud.query_count) AS total_queries, ROUND(SUM(pud.failed_query_count) * 100.0 / SUM(pud.query_count), 2) AS failure_rate FROM product_usage_daily pud JOIN feature_catalog fc ON pud.feature_id = fc.feature_id JOIN products p ON fc.product_id = p.product_id GROUP BY fc.feature_id, fc.feature_name, p.product_name ORDER BY total_failures DESC LIMIT 5","difficulty":"hard","category":"aggregation","expected_tables":["product_usage_daily","feature_catalog","products"]},
    {"id":"test_026","question":"Compare average deal size between inbound and outbound opportunities","expected_sql":"SELECT source, COUNT(*) AS deals, AVG(expected_arr) AS avg_deal_size, SUM(expected_arr) AS total_arr FROM opportunities WHERE source IN ('inbound','outbound') GROUP BY source","difficulty":"hard","category":"comparison","expected_tables":["opportunities"]},
    {"id":"test_027","question":"Show incident impact duration by severity","expected_sql":"SELECT severity, COUNT(*) AS incidents, AVG(TIMESTAMPDIFF(MINUTE, started_at, ended_at)) AS avg_duration_min FROM incidents WHERE ended_at IS NOT NULL GROUP BY severity ORDER BY FIELD(severity,'sev1','sev2','sev3')","difficulty":"hard","category":"aggregation","expected_tables":["incidents"]},
    {"id":"test_028","question":"Which accounts have the highest invoice delinquency?","expected_sql":"SELECT a.account_name, a.segment, COUNT(i.invoice_id) AS overdue_invoices, SUM(i.total) AS overdue_amount FROM invoices i JOIN accounts a ON i.account_id = a.account_id WHERE i.status IN ('open','uncollectible') AND i.due_date < CURDATE() GROUP BY a.account_id, a.account_name, a.segment ORDER BY overdue_amount DESC LIMIT 10","difficulty":"hard","category":"aggregation","expected_tables":["invoices","accounts"]},
    {"id":"test_029","question":"Show workspace user activity by role across environments","expected_sql":"SELECT w.environment, wu.role, COUNT(*) AS users, MAX(wu.last_active_at) AS most_recent_active FROM workspace_users wu JOIN workspaces w ON wu.workspace_id = w.workspace_id WHERE wu.is_service_account = FALSE GROUP BY w.environment, wu.role ORDER BY w.environment, users DESC","difficulty":"hard","category":"aggregation","expected_tables":["workspace_users","workspaces"]},
    {"id":"test_030","question":"Calculate the average time to first escalation for P1 tickets","expected_sql":"SELECT a.segment, AVG(TIMESTAMPDIFF(MINUTE, st.created_at, te.event_time)) AS avg_time_to_escalation_min FROM ticket_events te JOIN support_tickets st ON te.ticket_id = st.ticket_id JOIN accounts a ON st.account_id = a.account_id WHERE te.event_type = 'escalated' AND st.priority = 'P1' GROUP BY a.segment ORDER BY avg_time_to_escalation_min ASC","difficulty":"hard","category":"aggregation","expected_tables":["ticket_events","support_tickets","accounts"]},
    {"id":"test_031","question":"Show total revenue collected versus total invoiced by segment","expected_sql":"SELECT a.segment, SUM(i.total) AS total_invoiced, SUM(CASE WHEN i.status = 'paid' THEN i.total ELSE 0 END) AS total_collected, ROUND(SUM(CASE WHEN i.status = 'paid' THEN i.total ELSE 0 END) * 100.0 / SUM(i.total), 2) AS collection_rate FROM invoices i JOIN accounts a ON i.account_id = a.account_id GROUP BY a.segment ORDER BY collection_rate DESC","difficulty":"hard","category":"aggregation","expected_tables":["invoices","accounts"]},
    {"id":"test_032","question":"Find accounts with high usage but low contracted ARR","expected_sql":"SELECT a.account_name, a.segment, SUM(pud.query_count) AS total_queries, SUM(s.contracted_arr) AS total_arr FROM accounts a JOIN product_usage_daily pud ON a.account_id = pud.account_id JOIN subscriptions s ON a.account_id = s.account_id WHERE s.status = 'active' GROUP BY a.account_id, a.account_name, a.segment HAVING SUM(pud.query_count) > 10000 AND SUM(s.contracted_arr) < 50000 ORDER BY total_queries DESC","difficulty":"hard","category":"aggregation","expected_tables":["accounts","product_usage_daily","subscriptions"]},
    {"id":"test_033","question":"Show ticket escalation counts by assigned employee","expected_sql":"SELECT e.full_name, e.role, COUNT(te.event_id) AS escalations FROM ticket_events te JOIN support_tickets st ON te.ticket_id = st.ticket_id JOIN employees e ON st.assigned_employee_id = e.employee_id WHERE te.event_type = 'escalated' GROUP BY e.employee_id, e.full_name, e.role ORDER BY escalations DESC LIMIT 10","difficulty":"hard","category":"aggregation","expected_tables":["ticket_events","support_tickets","employees"]},
    {"id":"test_034","question":"What is the average deal cycle by opportunity stage for closed-won deals?","expected_sql":"SELECT forecast_category, AVG(DATEDIFF(close_date, created_date)) AS avg_cycle_days, COUNT(*) AS deals FROM opportunities WHERE stage = 'closed_won' GROUP BY forecast_category ORDER BY avg_cycle_days","difficulty":"hard","category":"aggregation","expected_tables":["opportunities"]},
    {"id":"test_035","question":"Show root causes of incidents with their frequency","expected_sql":"SELECT root_cause, COUNT(*) AS occurrences, GROUP_CONCAT(DISTINCT affected_product) AS products_affected FROM incidents WHERE root_cause IS NOT NULL GROUP BY root_cause ORDER BY occurrences DESC","difficulty":"medium","category":"aggregation","expected_tables":["incidents"]},
    {"id":"test_036","question":"List blocked queries from the audit log with error details","expected_sql":"SELECT query_text, intent, error_message, created_at FROM query_audit_log WHERE status = 'blocked' ORDER BY created_at DESC LIMIT 20","difficulty":"easy","category":"data_query","expected_tables":["query_audit_log"]},
    {"id":"test_037","question":"Show the top 10 accounts by employee count","expected_sql":"SELECT account_name, industry, segment, employee_count, region FROM accounts ORDER BY employee_count DESC LIMIT 10","difficulty":"easy","category":"data_query","expected_tables":["accounts"]},
    {"id":"test_038","question":"What is the distribution of subscription statuses?","expected_sql":"SELECT status, COUNT(*) AS count, SUM(contracted_arr) AS total_arr FROM subscriptions GROUP BY status ORDER BY count DESC","difficulty":"easy","category":"aggregation","expected_tables":["subscriptions"]},
    {"id":"test_039","question":"Show ticket events for escalated tickets with customer impact","expected_sql":"SELECT st.ticket_id, st.subject, st.priority, te.event_type, te.event_time, te.note FROM ticket_events te JOIN support_tickets st ON te.ticket_id = st.ticket_id WHERE te.event_type = 'escalated' ORDER BY te.event_time DESC LIMIT 20","difficulty":"medium","category":"data_query","expected_tables":["ticket_events","support_tickets"]},
    {"id":"test_040","question":"Compare ARR between North America and Europe","expected_sql":"SELECT a.region, COUNT(DISTINCT a.account_id) AS accounts, SUM(s.contracted_arr) AS total_arr FROM subscriptions s JOIN accounts a ON s.account_id = a.account_id WHERE s.status = 'active' AND a.region IN ('North America','Europe') GROUP BY a.region","difficulty":"medium","category":"comparison","expected_tables":["subscriptions","accounts"]}
]