backend/evaluation/dataset.json

[
    {
        "id": "eval_001",
        "question": "Show all accounts with high churn risk",
        "expected_sql": "SELECT account_name, industry, segment, region, health_score, churn_risk FROM accounts WHERE churn_risk = 'high' ORDER BY health_score ASC",
        "difficulty": "easy",
        "category": "data_query",
        "expected_tables": ["accounts"]
    },
    {
        "id": "eval_002",
        "question": "What is the total contracted ARR across all active subscriptions?",
        "expected_sql": "SELECT SUM(contracted_arr) AS total_arr FROM subscriptions WHERE status = 'active'",
        "difficulty": "easy",
        "category": "aggregation",
        "expected_tables": ["subscriptions"]
    },
    {
        "id": "eval_003",
        "question": "List all open P1 support tickets",
        "expected_sql": "SELECT ticket_id, subject, category, created_at FROM support_tickets WHERE priority = 'P1' AND status = 'open' ORDER BY created_at DESC",
        "difficulty": "easy",
        "category": "data_query",
        "expected_tables": ["support_tickets"]
    },
    {
        "id": "eval_004",
        "question": "Show employees hired in the last year",
        "expected_sql": "SELECT full_name, role, location, hire_date FROM employees WHERE hire_date >= DATE_SUB(CURDATE(), INTERVAL 1 YEAR) ORDER BY hire_date DESC",
        "difficulty": "easy",
        "category": "data_query",
        "expected_tables": ["employees"]
    },
    {
        "id": "eval_005",
        "question": "How many invoices are currently open?",
        "expected_sql": "SELECT COUNT(*) AS open_invoices FROM invoices WHERE status = 'open'",
        "difficulty": "easy",
        "category": "aggregation",
        "expected_tables": ["invoices"]
    },
    {
        "id": "eval_006",
        "question": "Show all enterprise segment accounts sorted by health score",
        "expected_sql": "SELECT account_name, industry, region, health_score, churn_risk FROM accounts WHERE segment = 'enterprise' ORDER BY health_score DESC",
        "difficulty": "easy",
        "category": "data_query",
        "expected_tables": ["accounts"]
    },
    {
        "id": "eval_007",
        "question": "List all plans and their monthly prices",
        "expected_sql": "SELECT plan_name, billing_model, monthly_base_price, included_seats, support_tier FROM plans ORDER BY monthly_base_price ASC",
        "difficulty": "easy",
        "category": "data_query",
        "expected_tables": ["plans"]
    },
    {
        "id": "eval_008",
        "question": "How many employees are in each department?",
        "expected_sql": "SELECT d.name AS department, COUNT(e.employee_id) AS headcount FROM departments d JOIN employees e ON d.department_id = e.department_id GROUP BY d.name ORDER BY headcount DESC",
        "difficulty": "easy",
        "category": "aggregation",
        "expected_tables": ["departments", "employees"]
    },
    {
        "id": "eval_009",
        "question": "Show all sev1 incidents from this year",
        "expected_sql": "SELECT incident_id, started_at, ended_at, affected_product, customer_impact, root_cause FROM incidents WHERE severity = 'sev1' AND YEAR(started_at) = YEAR(CURDATE()) ORDER BY started_at DESC",
        "difficulty": "easy",
        "category": "data_query",
        "expected_tables": ["incidents"]
    },
    {
        "id": "eval_010",
        "question": "What is the total payment amount by payment method?",
        "expected_sql": "SELECT payment_method, COUNT(*) AS payment_count, SUM(amount) AS total_amount FROM payments GROUP BY payment_method ORDER BY total_amount DESC",
        "difficulty": "easy",
        "category": "aggregation",
        "expected_tables": ["payments"]
    },
    {
        "id": "eval_011",
        "question": "Show the number of accounts by segment and region",
        "expected_sql": "SELECT segment, region, COUNT(*) AS account_count FROM accounts GROUP BY segment, region ORDER BY segment, account_count DESC",
        "difficulty": "easy",
        "category": "aggregation",
        "expected_tables": ["accounts"]
    },
    {
        "id": "eval_012",
        "question": "List all features in the catalog with their risk level",
        "expected_sql": "SELECT fc.feature_name, fc.feature_category, fc.risk_level, p.product_name FROM feature_catalog fc JOIN products p ON fc.product_id = p.product_id ORDER BY fc.risk_level DESC, fc.feature_name",
        "difficulty": "easy",
        "category": "data_query",
        "expected_tables": ["feature_catalog", "products"]
    },
    {
        "id": "eval_013",
        "question": "How many workspaces are in each environment type?",
        "expected_sql": "SELECT environment, COUNT(*) AS workspace_count FROM workspaces GROUP BY environment ORDER BY workspace_count DESC",
        "difficulty": "easy",
        "category": "aggregation",
        "expected_tables": ["workspaces"]
    },
    {
        "id": "eval_014",
        "question": "Show the annual budget by department",
        "expected_sql": "SELECT name, cost_center, region, annual_budget FROM departments ORDER BY annual_budget DESC",
        "difficulty": "easy",
        "category": "data_query",
        "expected_tables": ["departments"]
    },
    {
        "id": "eval_015",
        "question": "What is the average contracted ARR by plan?",
        "expected_sql": "SELECT p.plan_name, AVG(s.contracted_arr) AS avg_arr, COUNT(*) AS subscription_count FROM subscriptions s JOIN plans p ON s.plan_id = p.plan_id GROUP BY p.plan_name ORDER BY avg_arr DESC",
        "difficulty": "medium",
        "category": "aggregation",
        "expected_tables": ["subscriptions", "plans"]
    },
    {
        "id": "eval_016",
        "question": "Show total invoice revenue by account segment",
        "expected_sql": "SELECT a.segment, COUNT(i.invoice_id) AS invoice_count, SUM(i.total) AS total_revenue FROM invoices i JOIN accounts a ON i.account_id = a.account_id WHERE i.status = 'paid' GROUP BY a.segment ORDER BY total_revenue DESC",
        "difficulty": "medium",
        "category": "aggregation",
        "expected_tables": ["invoices", "accounts"]
    },
    {
        "id": "eval_017",
        "question": "Which accounts have the most open support tickets?",
        "expected_sql": "SELECT a.account_name, a.segment, COUNT(st.ticket_id) AS open_tickets FROM support_tickets st JOIN accounts a ON st.account_id = a.account_id WHERE st.status = 'open' GROUP BY a.account_id, a.account_name, a.segment ORDER BY open_tickets DESC LIMIT 10",
        "difficulty": "medium",
        "category": "aggregation",
        "expected_tables": ["support_tickets", "accounts"]
    },
    {
        "id": "eval_018",
        "question": "Show monthly invoice volume trends",
        "expected_sql": "SELECT DATE_FORMAT(invoice_date, '%Y-%m') AS month, COUNT(*) AS invoice_count, SUM(total) AS monthly_revenue FROM invoices GROUP BY DATE_FORMAT(invoice_date, '%Y-%m') ORDER BY DATE_FORMAT(invoice_date, '%Y-%m')",
        "difficulty": "medium",
        "category": "aggregation",
        "expected_tables": ["invoices"]
    },
    {
        "id": "eval_019",
        "question": "List opportunities in the negotiation stage with expected ARR above 500K",
        "expected_sql": "SELECT o.opportunity_name, a.account_name, o.expected_arr, o.probability_pct, o.close_date FROM opportunities o JOIN accounts a ON o.account_id = a.account_id WHERE o.stage = 'negotiation' AND o.expected_arr > 500000 ORDER BY o.expected_arr DESC",
        "difficulty": "medium",
        "category": "data_query",
        "expected_tables": ["opportunities", "accounts"]
    },
    {
        "id": "eval_020",
        "question": "What is the average first response time for P1 tickets by account segment?",
        "expected_sql": "SELECT a.segment, AVG(st.first_response_minutes) AS avg_first_response_min, COUNT(st.ticket_id) AS ticket_count FROM support_tickets st JOIN accounts a ON st.account_id = a.account_id WHERE st.priority = 'P1' GROUP BY a.segment ORDER BY avg_first_response_min ASC",
        "difficulty": "medium",
        "category": "aggregation",
        "expected_tables": ["support_tickets", "accounts"]
    },
    {
        "id": "eval_021",
        "question": "Show the top 10 workspaces by total query count",
        "expected_sql": "SELECT w.workspace_name, a.account_name, SUM(pud.query_count) AS total_queries, SUM(pud.failed_query_count) AS total_failures FROM product_usage_daily pud JOIN workspaces w ON pud.workspace_id = w.workspace_id JOIN accounts a ON pud.account_id = a.account_id GROUP BY w.workspace_id, w.workspace_name, a.account_name ORDER BY total_queries DESC LIMIT 10",
        "difficulty": "medium",
        "category": "aggregation",
        "expected_tables": ["product_usage_daily", "workspaces", "accounts"]
    },
    {
        "id": "eval_022",
        "question": "How many opportunities are in each pipeline stage?",
        "expected_sql": "SELECT stage, COUNT(*) AS opportunity_count, SUM(expected_arr) AS total_expected_arr, AVG(probability_pct) AS avg_probability FROM opportunities GROUP BY stage ORDER BY FIELD(stage, 'prospecting', 'qualification', 'technical_validation', 'proposal', 'negotiation', 'closed_won', 'closed_lost')",
        "difficulty": "medium",
        "category": "aggregation",
        "expected_tables": ["opportunities"]
    },
    {
        "id": "eval_023",
        "question": "Show the average CSAT score by ticket category",
        "expected_sql": "SELECT category, AVG(csat_score) AS avg_csat, COUNT(*) AS ticket_count FROM support_tickets WHERE csat_score IS NOT NULL GROUP BY category ORDER BY avg_csat DESC",
        "difficulty": "medium",
        "category": "aggregation",
        "expected_tables": ["support_tickets"]
    },
    {
        "id": "eval_024",
        "question": "List accounts with active subscriptions that are past due",
        "expected_sql": "SELECT a.account_name, a.segment, s.contracted_arr, s.seats_purchased, p.plan_name FROM subscriptions s JOIN accounts a ON s.account_id = a.account_id JOIN plans p ON s.plan_id = p.plan_id WHERE s.status = 'past_due' ORDER BY s.contracted_arr DESC",
        "difficulty": "medium",
        "category": "data_query",
        "expected_tables": ["subscriptions", "accounts", "plans"]
    },
    {
        "id": "eval_025",
        "question": "What is the success rate of queries by route type?",
        "expected_sql": "SELECT route, COUNT(*) AS total_queries, SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) AS successful, ROUND(SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 2) AS success_rate FROM query_audit_log GROUP BY route ORDER BY success_rate DESC",
        "difficulty": "medium",
        "category": "aggregation",
        "expected_tables": ["query_audit_log"]
    },
    {
        "id": "eval_026",
        "question": "Show the total contracted ARR by customer segment and region",
        "expected_sql": "SELECT a.segment, a.region, COUNT(s.subscription_id) AS subscriptions, SUM(s.contracted_arr) AS total_arr FROM subscriptions s JOIN accounts a ON s.account_id = a.account_id WHERE s.status = 'active' GROUP BY a.segment, a.region ORDER BY total_arr DESC",
        "difficulty": "medium",
        "category": "aggregation",
        "expected_tables": ["subscriptions", "accounts"]
    },
    {
        "id": "eval_027",
        "question": "Which departments have the highest total salary expense?",
        "expected_sql": "SELECT d.name AS department, COUNT(e.employee_id) AS headcount, SUM(e.base_salary) AS total_salary, AVG(e.base_salary) AS avg_salary FROM departments d JOIN employees e ON d.department_id = e.department_id WHERE e.active = TRUE GROUP BY d.name ORDER BY total_salary DESC",
        "difficulty": "medium",
        "category": "aggregation",
        "expected_tables": ["departments", "employees"]
    },
    {
        "id": "eval_028",
        "question": "Show contacts who are executive sponsors at enterprise accounts",
        "expected_sql": "SELECT c.full_name, c.title, c.department, a.account_name, a.segment FROM contacts c JOIN accounts a ON c.account_id = a.account_id WHERE c.is_executive_sponsor = TRUE AND a.segment IN ('enterprise', 'strategic') ORDER BY a.account_name",
        "difficulty": "medium",
        "category": "data_query",
        "expected_tables": ["contacts", "accounts"]
    },
    {
        "id": "eval_029",
        "question": "What are the top loss reasons for closed-lost opportunities?",
        "expected_sql": "SELECT loss_reason, COUNT(*) AS count, SUM(expected_arr) AS lost_arr FROM opportunities WHERE stage = 'closed_lost' AND loss_reason IS NOT NULL GROUP BY loss_reason ORDER BY count DESC",
        "difficulty": "medium",
        "category": "aggregation",
        "expected_tables": ["opportunities"]
    },
    {
        "id": "eval_030",
        "question": "Show the average discount percentage by plan type",
        "expected_sql": "SELECT p.plan_name, AVG(s.discount_pct) AS avg_discount, COUNT(*) AS subscription_count FROM subscriptions s JOIN plans p ON s.plan_id = p.plan_id GROUP BY p.plan_name ORDER BY avg_discount DESC",
        "difficulty": "medium",
        "category": "aggregation",
        "expected_tables": ["subscriptions", "plans"]
    },
    {
        "id": "eval_031",
        "question": "Compare ticket volume and resolution time between P1 and P2 priorities",
        "expected_sql": "SELECT priority, COUNT(*) AS ticket_count, AVG(first_response_minutes) AS avg_response_min, AVG(resolution_minutes) AS avg_resolution_min FROM support_tickets WHERE priority IN ('P1', 'P2') GROUP BY priority",
        "difficulty": "medium",
        "category": "comparison",
        "expected_tables": ["support_tickets"]
    },
    {
        "id": "eval_032",
        "question": "Which sales reps own the most subscription ARR?",
        "expected_sql": "SELECT e.full_name, e.role, COUNT(s.subscription_id) AS subscription_count, SUM(s.contracted_arr) AS total_arr FROM subscriptions s JOIN employees e ON s.owner_employee_id = e.employee_id WHERE s.status = 'active' GROUP BY e.employee_id, e.full_name, e.role ORDER BY total_arr DESC LIMIT 10",
        "difficulty": "medium",
        "category": "aggregation",
        "expected_tables": ["subscriptions", "employees"]
    },
    {
        "id": "eval_033",
        "question": "Show daily product usage trends for the last 30 days",
        "expected_sql": "SELECT usage_date, SUM(active_users) AS total_active_users, SUM(query_count) AS total_queries, SUM(failed_query_count) AS total_failures FROM product_usage_daily WHERE usage_date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY) GROUP BY usage_date ORDER BY usage_date",
        "difficulty": "medium",
        "category": "aggregation",
        "expected_tables": ["product_usage_daily"]
    },
    {
        "id": "eval_034",
        "question": "Find accounts with high ARR but low health scores",
        "expected_sql": "SELECT a.account_name, a.segment, a.health_score, a.churn_risk, SUM(s.contracted_arr) AS total_arr FROM accounts a JOIN subscriptions s ON a.account_id = s.account_id WHERE s.status = 'active' AND a.health_score < 50 GROUP BY a.account_id, a.account_name, a.segment, a.health_score, a.churn_risk HAVING SUM(s.contracted_arr) > 100000 ORDER BY total_arr DESC",
        "difficulty": "hard",
        "category": "aggregation",
        "expected_tables": ["accounts", "subscriptions"]
    },
    {
        "id": "eval_035",
        "question": "Calculate the payment failure rate by account segment",
        "expected_sql": "SELECT a.segment, COUNT(*) AS total_payments, SUM(CASE WHEN pay.payment_status = 'failed' THEN 1 ELSE 0 END) AS failed_payments, ROUND(SUM(CASE WHEN pay.payment_status = 'failed' THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 2) AS failure_rate FROM payments pay JOIN accounts a ON pay.account_id = a.account_id GROUP BY a.segment ORDER BY failure_rate DESC",
        "difficulty": "hard",
        "category": "aggregation",
        "expected_tables": ["payments", "accounts"]
    },
    {
        "id": "eval_036",
        "question": "Show accounts where churn risk is high and they have unresolved P1 tickets",
        "expected_sql": "SELECT DISTINCT a.account_name, a.segment, a.health_score, a.churn_risk, st.subject, st.created_at FROM accounts a JOIN support_tickets st ON a.account_id = st.account_id WHERE a.churn_risk = 'high' AND st.priority = 'P1' AND st.status IN ('open', 'pending_customer') ORDER BY a.health_score ASC",
        "difficulty": "hard",
        "category": "data_query",
        "expected_tables": ["accounts", "support_tickets"]
    },
    {
        "id": "eval_037",
        "question": "Calculate net revenue retention by customer segment",
        "expected_sql": "SELECT a.segment, SUM(CASE WHEN s.status = 'active' THEN s.contracted_arr ELSE 0 END) AS current_arr, SUM(s.contracted_arr) AS original_arr, ROUND(SUM(CASE WHEN s.status = 'active' THEN s.contracted_arr ELSE 0 END) * 100.0 / SUM(s.contracted_arr), 2) AS nrr_pct FROM subscriptions s JOIN accounts a ON s.account_id = a.account_id GROUP BY a.segment ORDER BY nrr_pct DESC",
        "difficulty": "hard",
        "category": "aggregation",
        "expected_tables": ["subscriptions", "accounts"]
    },
    {
        "id": "eval_038",
        "question": "Rank sales reps by closed-won opportunity ARR and average deal cycle length",
        "expected_sql": "SELECT e.full_name, COUNT(o.opportunity_id) AS deals_won, SUM(o.expected_arr) AS total_arr, AVG(DATEDIFF(o.close_date, o.created_date)) AS avg_cycle_days FROM opportunities o JOIN employees e ON o.owner_employee_id = e.employee_id WHERE o.stage = 'closed_won' GROUP BY e.employee_id, e.full_name ORDER BY total_arr DESC",
        "difficulty": "hard",
        "category": "aggregation",
        "expected_tables": ["opportunities", "employees"]
    },
    {
        "id": "eval_039",
        "question": "Show the correlation between product usage and support ticket volume per account",
        "expected_sql": "SELECT a.account_name, a.segment, COALESCE(SUM(pud.query_count), 0) AS total_queries, COALESCE(COUNT(DISTINCT st.ticket_id), 0) AS ticket_count FROM accounts a LEFT JOIN product_usage_daily pud ON a.account_id = pud.account_id LEFT JOIN support_tickets st ON a.account_id = st.account_id GROUP BY a.account_id, a.account_name, a.segment ORDER BY ticket_count DESC LIMIT 20",
        "difficulty": "hard",
        "category": "aggregation",
        "expected_tables": ["accounts", "product_usage_daily", "support_tickets"]
    },
    {
        "id": "eval_040",
        "question": "Which accounts have subscriptions but zero product usage in the last 90 days?",
        "expected_sql": "SELECT a.account_name, a.segment, s.contracted_arr FROM accounts a JOIN subscriptions s ON a.account_id = s.account_id WHERE s.status = 'active' AND a.account_id NOT IN (SELECT DISTINCT account_id FROM product_usage_daily WHERE usage_date >= DATE_SUB(CURDATE(), INTERVAL 90 DAY)) ORDER BY s.contracted_arr DESC",
        "difficulty": "hard",
        "category": "data_query",
        "expected_tables": ["accounts", "subscriptions", "product_usage_daily"]
    },
    {
        "id": "eval_041",
        "question": "Show quarterly invoice revenue trends with quarter-over-quarter growth",
        "expected_sql": "SELECT YEAR(invoice_date) AS yr, QUARTER(invoice_date) AS qtr, SUM(total) AS quarterly_revenue FROM invoices WHERE status = 'paid' GROUP BY YEAR(invoice_date), QUARTER(invoice_date) ORDER BY YEAR(invoice_date), QUARTER(invoice_date)",
        "difficulty": "hard",
        "category": "aggregation",
        "expected_tables": ["invoices"]
    },
    {
        "id": "eval_042",
        "question": "Find the average time between ticket creation and resolution by priority",
        "expected_sql": "SELECT priority, COUNT(*) AS resolved_tickets, AVG(resolution_minutes) AS avg_resolution_min, MIN(resolution_minutes) AS fastest_min, MAX(resolution_minutes) AS slowest_min FROM support_tickets WHERE resolution_minutes IS NOT NULL GROUP BY priority ORDER BY FIELD(priority, 'P1', 'P2', 'P3', 'P4')",
        "difficulty": "hard",
        "category": "aggregation",
        "expected_tables": ["support_tickets"]
    },
    {
        "id": "eval_043",
        "question": "Show the pipeline forecast by forecast category with weighted ARR",
        "expected_sql": "SELECT forecast_category, COUNT(*) AS deal_count, SUM(expected_arr) AS total_arr, SUM(expected_arr * probability_pct / 100) AS weighted_arr FROM opportunities WHERE stage NOT IN ('closed_won', 'closed_lost') GROUP BY forecast_category ORDER BY FIELD(forecast_category, 'commit', 'best_case', 'pipeline')",
        "difficulty": "hard",
        "category": "aggregation",
        "expected_tables": ["opportunities"]
    },
    {
        "id": "eval_044",
        "question": "Which workspace users are least active in production environments?",
        "expected_sql": "SELECT wu.email, wu.role, w.workspace_name, a.account_name, wu.last_active_at FROM workspace_users wu JOIN workspaces w ON wu.workspace_id = w.workspace_id JOIN accounts a ON wu.account_id = a.account_id WHERE w.environment = 'production' AND wu.is_service_account = FALSE ORDER BY wu.last_active_at ASC LIMIT 20",
        "difficulty": "hard",
        "category": "data_query",
        "expected_tables": ["workspace_users", "workspaces", "accounts"]
    },
    {
        "id": "eval_045",
        "question": "Calculate the SLA breach rate for P1 tickets by account",
        "expected_sql": "SELECT a.account_name, a.segment, COUNT(te.event_id) AS total_breaches FROM ticket_events te JOIN support_tickets st ON te.ticket_id = st.ticket_id JOIN accounts a ON st.account_id = a.account_id WHERE te.event_type = 'sla_breach' AND st.priority = 'P1' GROUP BY a.account_id, a.account_name, a.segment ORDER BY total_breaches DESC LIMIT 10",
        "difficulty": "hard",
        "category": "aggregation",
        "expected_tables": ["ticket_events", "support_tickets", "accounts"]
    },
    {
        "id": "eval_046",
        "question": "Show the average p95 latency by feature and product",
        "expected_sql": "SELECT p.product_name, fc.feature_name, AVG(pud.p95_latency_ms) AS avg_p95_latency, SUM(pud.query_count) AS total_queries FROM product_usage_daily pud JOIN feature_catalog fc ON pud.feature_id = fc.feature_id JOIN products p ON fc.product_id = p.product_id GROUP BY p.product_name, fc.feature_name ORDER BY avg_p95_latency DESC",
        "difficulty": "hard",
        "category": "aggregation",
        "expected_tables": ["product_usage_daily", "feature_catalog", "products"]
    },
    {
        "id": "eval_047",
        "question": "Find stalled opportunities that haven't had activity in 30 days",
        "expected_sql": "SELECT o.opportunity_name, a.account_name, o.stage, o.expected_arr, o.last_activity_date, DATEDIFF(CURDATE(), o.last_activity_date) AS days_stalled FROM opportunities o JOIN accounts a ON o.account_id = a.account_id WHERE o.stage NOT IN ('closed_won', 'closed_lost') AND o.last_activity_date < DATE_SUB(CURDATE(), INTERVAL 30 DAY) ORDER BY o.expected_arr DESC",
        "difficulty": "hard",
        "category": "data_query",
        "expected_tables": ["opportunities", "accounts"]
    },
    {
        "id": "eval_048",
        "question": "Show total compute cost by account with subscription context",
        "expected_sql": "SELECT a.account_name, a.segment, SUM(pud.compute_seconds) AS total_compute_sec, SUM(s.contracted_arr) AS total_arr FROM product_usage_daily pud JOIN accounts a ON pud.account_id = a.account_id JOIN subscriptions s ON a.account_id = s.account_id WHERE s.status = 'active' GROUP BY a.account_id, a.account_name, a.segment ORDER BY total_compute_sec DESC LIMIT 15",
        "difficulty": "hard",
        "category": "aggregation",
        "expected_tables": ["product_usage_daily", "accounts", "subscriptions"]
    },
    {
        "id": "eval_049",
        "question": "What percentage of total ARR is at risk from accounts with high churn risk?",
        "expected_sql": "SELECT a.churn_risk, COUNT(DISTINCT a.account_id) AS account_count, SUM(s.contracted_arr) AS arr_at_risk, ROUND(SUM(s.contracted_arr) * 100.0 / (SELECT SUM(contracted_arr) FROM subscriptions WHERE status = 'active'), 2) AS pct_of_total_arr FROM accounts a JOIN subscriptions s ON a.account_id = s.account_id WHERE s.status = 'active' GROUP BY a.churn_risk ORDER BY FIELD(a.churn_risk, 'high', 'medium', 'low')",
        "difficulty": "hard",
        "category": "aggregation",
        "expected_tables": ["accounts", "subscriptions"]
    },
    {
        "id": "eval_050",
        "question": "Show the top revenue-generating accounts with their support ticket count and average CSAT",
        "expected_sql": "SELECT a.account_name, a.segment, SUM(s.contracted_arr) AS total_arr, COUNT(DISTINCT st.ticket_id) AS ticket_count, AVG(st.csat_score) AS avg_csat FROM accounts a JOIN subscriptions s ON a.account_id = s.account_id LEFT JOIN support_tickets st ON a.account_id = st.account_id WHERE s.status = 'active' GROUP BY a.account_id, a.account_name, a.segment ORDER BY total_arr DESC LIMIT 10",
        "difficulty": "hard",
        "category": "aggregation",
        "expected_tables": ["accounts", "subscriptions", "support_tickets"]
    },
    {
        "id": "eval_051",
        "question": "Show net revenue retention by customer segment for the last 4 quarters",
        "expected_sql": "SELECT a.segment, YEAR(s.start_date) AS yr, QUARTER(s.start_date) AS qtr, SUM(CASE WHEN s.status = 'active' THEN s.contracted_arr ELSE 0 END) AS active_arr, SUM(s.contracted_arr) AS total_arr, ROUND(SUM(CASE WHEN s.status = 'active' THEN s.contracted_arr ELSE 0 END) * 100.0 / SUM(s.contracted_arr), 2) AS nrr_pct FROM subscriptions s JOIN accounts a ON s.account_id = a.account_id WHERE s.start_date >= DATE_SUB((SELECT MAX(start_date) FROM subscriptions), INTERVAL 1 YEAR) GROUP BY a.segment, YEAR(s.start_date), QUARTER(s.start_date) ORDER BY yr DESC, qtr DESC, nrr_pct DESC",
        "difficulty": "hard",
        "category": "aggregation",
        "expected_tables": ["accounts", "subscriptions"]
    }
]