evaluation/development-gpu.jsonl

{"id": "2e299c717e66247d2e4b_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "missing_schema", "question": "I didn't provide schema. Please find the available tables before writing sql for repairs.", "expected": {"action": "call", "name": "jupmrn_list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"jupmrn_list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a2ad743e655e7c8923dc_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "project", "question": "bring up customer_name from rentals only", "expected": {"action": "call", "name": "vgeihrmq.execute_sql", "arguments": {"query": "SELECT customer_name FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"vgeihrmq.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7513979dd6549473de83_hinglish__development_layout", "language": "hinglish", "backend": "supabase", "operation": "count_eq", "question": "rentals mein city active wale data ki sankhya batayo.", "expected": {"action": "call", "name": "dhofp__execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM rentals WHERE city = 'active';"}}, "output": "{\"action\":\"call\",\"name\":\"dhofp__execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM rentals WHERE city = 'active';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c23d51b5c96bb4b300bf_en__development_layout", "language": "en", "backend": "supabase", "operation": "group_count", "question": "Sum up the rows for each status in rentals.", "expected": {"action": "call", "name": "zyaxplyd__execute_sql", "arguments": {"query": "SELECT status, COUNT(*) FROM rentals GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"zyaxplyd__execute_sql\",\"arguments\":{\"query\":\"SELECT status, COUNT(*) FROM rentals GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a7d907a9a18c081961a8_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "distinct", "question": "find the unique status in enrollments", "expected": {"action": "call", "name": "ygqgqmfznb__execute_sql", "arguments": {"query": "SELECT DISTINCT status FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"ygqgqmfznb__execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT status FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6016d266fd2535bc4479_en__development_layout", "language": "en", "backend": "supabase", "operation": "year", "question": "Display deliveries entries where order_date has the year 2021.", "expected": {"action": "call", "name": "geqwktokw__execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE EXTRACT(YEAR FROM order_date) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"geqwktokw__execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE EXTRACT(YEAR FROM order_date) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4658c817cf80ab71fd79_en__development_layout", "language": "en", "backend": "supabase", "operation": "max", "question": "Give me the highest rating for rentals.", "expected": {"action": "call", "name": "mxkvphj.execute_sql", "arguments": {"query": "SELECT MAX(rating) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"mxkvphj.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(rating) FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "de15f8143029f00970f6_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "bottom", "question": "give me the 1 lowest price data from deliveries ordered ascending", "expected": {"action": "call", "name": "pysnshmc_execute_sql", "arguments": {"query": "SELECT * FROM deliveries ORDER BY price ASC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"pysnshmc_execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries ORDER BY price ASC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7a57ce9685f252c72f44_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "lt", "question": "output the records from rentals where quantity is below 3 please.", "expected": {"action": "call", "name": "nlniybyheyhui_execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE quantity < 3;"}}, "output": "{\"action\":\"call\",\"name\":\"nlniybyheyhui_execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE quantity < 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f826d550b5dd9bf82144_hinglish__development_layout", "language": "hinglish", "backend": "supabase", "operation": "avg", "question": "repairs me rating ka average kya hai?", "expected": {"action": "call", "name": "uulxyowkdlozf.execute_sql", "arguments": {"query": "SELECT AVG(rating) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"uulxyowkdlozf.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(rating) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "327bced9ccce8adafb67_en__development_layout", "language": "en", "backend": "supabase", "operation": "lt", "question": "Present the data from enrollments where price is below 20.", "expected": {"action": "call", "name": "wwcdxkxwwy.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE price < 20;"}}, "output": "{\"action\":\"call\",\"name\":\"wwcdxkxwwy.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE price < 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "da7c6370afcc35f790f8_hinglish__development_layout", "language": "hinglish", "backend": "supabase", "operation": "sum", "question": "rentals mein quantity ka total measure karo.", "expected": {"action": "call", "name": "mhcjguia__execute_sql", "arguments": {"query": "SELECT SUM(quantity) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"mhcjguia__execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(quantity) FROM rentals WHERE kind = 'Jaipur';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "36d48e67b84ab1dd10fa_en__development_layout", "language": "en", "backend": "supabase", "operation": "project", "question": "Let's look at display_name in enrollments specifically.", "expected": {"action": "call", "name": "occvdoidmz__execute_sql", "arguments": {"query": "SELECT display_name FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"occvdoidmz__execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM enrollments WHERE kind = 'Petro';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "no such column: kind"}}
{"id": "39205adac55e98cd6749_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "project_eq", "question": "rentals से display_name आउटपुट करें जहाँ kind O'Reilly है।", "expected": {"action": "call", "name": "qgnefcu__execute_sql", "arguments": {"query": "SELECT display_name FROM rentals WHERE kind = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"qgnefcu__execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM rentals WHERE kind = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5ec7a0dca8e2804dbca9_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "not_null", "question": "Show the enrollments rows where item_name is not null.", "expected": {"action": "call", "name": "hzwaldfe.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE item_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"hzwaldfe.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE item_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ab37d15a3c7b3a27c08f_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "or", "question": "Please show rows from repairs where status is either South or active.", "expected": {"action": "call", "name": "skjfkkszk_execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE status IN ('South', 'active');"}}, "output": "{\"action\":\"call\",\"name\":\"skjfkkszk_execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE status IN ('South', 'active');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a7d907a9a18c081961a8_en__development_layout", "language": "en", "backend": "supabase", "operation": "distinct", "question": "Pull out the distinct status values from the data in enrollments.", "expected": {"action": "call", "name": "xeepsemwqpbmo__execute_sql", "arguments": {"query": "SELECT DISTINCT status FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"xeepsemwqpbmo__execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT status FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a8b04829d8a55fb6e012_en__development_layout", "language": "en", "backend": "supabase", "operation": "date_after", "question": "Query rentals for rows where recorded_at is at or after 2024-03-15.", "expected": {"action": "call", "name": "qhdkadzjih__execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE recorded_at >= '2024-03-15';"}}, "output": "{\"action\":\"call\",\"name\":\"qhdkadzjih__execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE recorded_at >= '2024-03-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c4e9855d27b398c547a8_hinglish__development_layout", "language": "hinglish", "backend": "supabase", "operation": "top", "question": "rentals mein balance ke highest values wali 1 rows display karo.", "expected": {"action": "call", "name": "fcesevlnzt__execute_sql", "arguments": {"query": "SELECT * FROM rentals ORDER BY balance DESC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"fcesevlnzt__execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals ORDER BY balance DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1e54bf2ef5c5da89e926_hinglish__development_layout", "language": "hinglish", "backend": "supabase", "operation": "having", "question": "enrollments se 1500 rows se zyada wale category groups ko unki row count ke saath dikhaiye.", "expected": {"action": "call", "name": "rkkuiasgy.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM enrollments GROUP BY category HAVING COUNT(*) > 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"rkkuiasgy.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM enrollments GROUP BY category HAVING COUNT(*) > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8e0d860cde6e650084c2_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "or", "question": "enrollments से kind Electronics या Mumbai वाली पंक्तियाँ लें।", "expected": {"action": "call", "name": "clradrusdix__execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE kind IN ('Electronics', 'Mumbai');"}}, "output": "{\"action\":\"call\",\"name\":\"clradrusdix__execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE kind IN ('Electronics', 'Mumbai');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f39ec09a8da50e72bdd5_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "min", "question": "enrollments में price का न्यूनतम स्तर क्या है?", "expected": {"action": "call", "name": "gnjyipsrev.execute_sql", "arguments": {"query": "SELECT MIN(price) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"gnjyipsrev.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(price) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a81aea4a2431376817c8_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "or", "question": "Please show rows from enrollments where kind is either Mumbai or Mumbai.", "expected": {"action": "call", "name": "rmhslrewbmk__execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE kind IN ('Mumbai', 'Mumbai');"}}, "output": "{\"action\":\"call\",\"name\":\"rmhslrewbmk__execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE kind IN ('Mumbai', 'Mumbai');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f826d550b5dd9bf82144_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "avg", "question": "repairs में rating का औसत क्या है?", "expected": {"action": "call", "name": "kixrdmdayyb__execute_sql", "arguments": {"query": "SELECT AVG(rating) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"kixrdmdayyb__execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(rating) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5fc77c83422c0e74572f_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "gt", "question": "look up the rows in deliveries where price is greater than 5000", "expected": {"action": "call", "name": "trkhbkcy.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE price > 5000;"}}, "output": "{\"action\":\"call\",\"name\":\"trkhbkcy.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE price > 5000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "07beb228a10fcdc90b00_en__development_layout", "language": "en", "backend": "mysql", "operation": "missing_schema", "question": "Please determine the available tables before writing SQL for enrollments, since I did not provide a database schema.", "expected": {"action": "call", "name": "nsnjtaz.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"nsnjtaz.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "40a4c97e6c006d30b593_en__development_layout", "language": "en", "backend": "supabase", "operation": "count", "question": "List the number of rows found in enrollments.", "expected": {"action": "call", "name": "ccpxaacd__execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT COUNT(*) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"ccpxaacd__execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT COUNT(*) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "64405609b3558aed49bd_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "date_after", "question": "Show me deliveries data where order_date is not earlier than 2025-01-15.", "expected": {"action": "call", "name": "brhoeoewlorpj.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE order_date >= '2025-01-15';"}}, "output": "{\"action\":\"call\",\"name\":\"brhoeoewlorpj.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE order_date >= '2025-01-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3800c9ec3cd53ff4c190_hinglish__development_layout", "language": "hinglish", "backend": "supabase", "operation": "year", "question": "2021 ke recorded_at ke hisaab se deliveries se data nikalo.", "expected": {"action": "call", "name": "ceole_execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE EXTRACT(YEAR FROM recorded_at) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"ceole_execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ab37d15a3c7b3a27c08f_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "or", "question": "repairs से status South या active वाली पंक्तियाँ लें।", "expected": {"action": "call", "name": "npczlgtzgcn.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE status IN ('South', 'active');"}}, "output": "{\"action\":\"call\",\"name\":\"npczlgtzgcn.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE status IN ('South', 'active');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9bb785183a3dc77c2cfa_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "group_sum", "question": "मुझे enrollments में प्रत्येक status के लिए balance का कुल दिखाना है।", "expected": {"action": "call", "name": "dpyhfdsea.execute_sql", "arguments": {"query": "SELECT status, SUM(balance) FROM enrollments GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"dpyhfdsea.execute_sql\",\"arguments\":{\"query\":\"SELECT status, SUM(balance) FROM enrollments GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3cfa2682031af21eedd8_hinglish__development_layout", "language": "hinglish", "backend": "supabase", "operation": "project", "question": "i want just name from deliveries ok", "expected": {"action": "call", "name": "tiqrxznewb__execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT name FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"tiqrxznewb__execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT name FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a888840d69a23410b604_en__development_layout", "language": "en", "backend": "supabase", "operation": "top", "question": "Present the top 5 records from repairs by descending quantity value.", "expected": {"action": "call", "name": "nysnejurog.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs ORDER BY quantity DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"nysnejurog.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs ORDER BY quantity DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f638660fca2c9a9c9386_en__development_layout", "language": "en", "backend": "supabase", "operation": "max", "question": "Give me the highest salary for enrollments.", "expected": {"action": "call", "name": "aqswgvdulg.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT MAX(salary) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"aqswgvdulg.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT MAX(salary) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "99327f4048c296f053ea_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "date_after", "question": "Show me enrollments data where created_at is not earlier than 2024-12-15.", "expected": {"action": "call", "name": "dqlhznwtcpqt.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE created_at >= '2024-12-15';"}}, "output": "{\"action\":\"call\",\"name\":\"dqlhznwtcpqt.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE created_at >= '2024-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a8b04829d8a55fb6e012_hinglish__development_layout", "language": "hinglish", "backend": "supabase", "operation": "date_after", "question": "rentals se wo records dikhao jahan recorded_at 2024-03-15 se pehle nahi hai.", "expected": {"action": "call", "name": "ivmkylimy_execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE recorded_at >= '2024-03-15';"}}, "output": "{\"action\":\"call\",\"name\":\"ivmkylimy_execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE recorded_at >= '2024-03-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "619f53e951d6c4a5acdb_en__development_layout", "language": "en", "backend": "supabase", "operation": "bottom", "question": "Pull the 1 rows with the lowest quantity scores from deliveries, smallest to largest.", "expected": {"action": "call", "name": "worotc_execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries ORDER BY quantity ASC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"worotc_execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries ORDER BY quantity ASC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e6b1d053d0a307dba955_hinglish__development_layout", "language": "hinglish", "backend": "supabase", "operation": "or", "question": "rentals se department Books ya North wali rows le lo.", "expected": {"action": "call", "name": "htvfqsv__execute_sql", "arguments": {"project_id": "demo_rentals", "query": "SELECT * FROM rentals WHERE department IN ('Books', 'North');"}}, "output": "{\"action\":\"call\",\"name\":\"htvfqsv__execute_sql\",\"arguments\":{\"project_id\":\"demo_rentals\",\"query\":\"SELECT * FROM rentals WHERE department IN ('Books', 'North');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f9c2698db9e7dbe6ec8e_en__development_layout", "language": "en", "backend": "supabase", "operation": "date_after", "question": "Query enrollments for rows where start_date is at or after 2025-09-15.", "expected": {"action": "call", "name": "bghzzliovs.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE start_date >= '2025-09-15';"}}, "output": "{\"action\":\"call\",\"name\":\"bghzzliovs.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE start_date >= '2025-09-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2b0c6cddd8771d41fc37_hinglish__development_layout", "language": "hinglish", "backend": "supabase", "operation": "count_eq", "question": "repairs mein department Pune wale data ki sankhya batayo.", "expected": {"action": "call", "name": "iqsfjxs__execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT COUNT(*) FROM repairs WHERE department = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"iqsfjxs__execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT COUNT(*) FROM repairs WHERE department = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bc42ac714674eb7e9508_hinglish__development_layout", "language": "hinglish", "backend": "supabase", "operation": "sum", "question": "repairs mein quantity ka total measure karo.", "expected": {"action": "call", "name": "oyeoidnlb.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT SUM(quantity) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"oyeoidnlb.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT SUM(quantity) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2e299c717e66247d2e4b_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "missing_schema", "question": "मेने डेटाबेस स्कीमा प्रदान नहीं की है, इसलिए repairs के लिए SQL लिखने से पहले उपलब्ध तालिकाओं को खोजें।", "expected": {"action": "call", "name": "dccvsxfgc__list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"dccvsxfgc__list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "60b3e2da11c7ba5f35b3_hinglish__development_layout", "language": "hinglish", "backend": "supabase", "operation": "year", "question": "2024 ke recorded_at ke hisaab se repairs se data nikalo.", "expected": {"action": "call", "name": "vluviexleyqn.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE EXTRACT(YEAR FROM recorded_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"vluviexleyqn.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE EXTRACT(YEAR FROM recorded_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6391074b92a18a2d8964_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "project", "question": "i want just display_name from enrollments ok", "expected": {"action": "call", "name": "nqoczsk.run_query", "arguments": {"query": "SELECT display_name FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"nqoczsk.run_query\",\"arguments\":{\"query\":\"SELECT display_name FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c077714e2e488a364925_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "between", "question": "show me the data from enrollments where quantity is between 5 and 15 inclusive of the boundaries", "expected": {"action": "call", "name": "stegkbawnxaj_execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE quantity BETWEEN 5 AND 15;"}}, "output": "{\"action\":\"call\",\"name\":\"stegkbawnxaj_execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE quantity BETWEEN 5 AND 15;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "619f53e951d6c4a5acdb_hinglish__development_layout", "language": "hinglish", "backend": "supabase", "operation": "bottom", "question": "deliveries se quantity ke hisaab se 1 minimum rows, lowest value se shuru karte hue, display karo.", "expected": {"action": "call", "name": "ljvagy.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries ORDER BY quantity ASC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"ljvagy.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries ORDER BY quantity ASC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3800c9ec3cd53ff4c190_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "year", "question": "2021 के recorded_at के आधार पर deliveries से डेटा निकालें।", "expected": {"action": "call", "name": "ufyso__execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE EXTRACT(YEAR FROM recorded_at) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"ufyso__execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6230fff878aa525542cd_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "missing_schema", "question": "Maine database schema provide nahi kiya hai, isliye enrollments ke liye SQL likhne se pehle available tables find karo.", "expected": {"action": "call", "name": "mjataawbatjn__list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"mjataawbatjn__list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "15e384ce46e890333204_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "gt", "question": "look up the rows in enrollments where rating is greater than 500", "expected": {"action": "call", "name": "cyrsqmdeybsw_execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE rating > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"cyrsqmdeybsw_execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE rating > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "99c40dbbcc3778d58436_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "group_count", "question": "rentals में प्रत्येक status के पंक्ति काउंट दें।", "expected": {"action": "call", "name": "zxzziqqemm.execute_sql", "arguments": {"query": "SELECT status, COUNT(*) FROM rentals GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"zxzziqqemm.execute_sql\",\"arguments\":{\"query\":\"SELECT status, COUNT(*) FROM rentals GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "640719c646d2ad1c92e6_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "sort_asc", "question": "rentals ki saari rows rating ke kam se zyada order mein sort karo.", "expected": {"action": "call", "name": "lndis.execute_sql", "arguments": {"sql": "SELECT * FROM rentals ORDER BY rating ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"lndis.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM rentals ORDER BY rating ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c077714e2e488a364925_hinglish__development_layout", "language": "hinglish", "backend": "supabase", "operation": "between", "question": "enrollments se wo rows show karo jahan quantity >= 5 aur <= 15 hai.", "expected": {"action": "call", "name": "vugeckvli_execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE quantity BETWEEN 5 AND 15;"}}, "output": "{\"action\":\"call\",\"name\":\"vugeckvli_execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE quantity BETWEEN 5 AND 15;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6d9c843a92d3ec7fb99c_en__development_layout", "language": "en", "backend": "mysql", "operation": "sort_desc", "question": "Return all rows from rentals with score in descending order.", "expected": {"action": "call", "name": "gnbuq.query", "arguments": {"query": "SELECT * FROM rentals ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"gnbuq.query\",\"arguments\":{\"query\":\"SELECT * FROM rentals ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "56594767710f98b9b347_hinglish__development_layout", "language": "hinglish", "backend": "supabase", "operation": "between", "question": "repairs se wo rows show karo jahan rating >= 5 aur <= 15 hai.", "expected": {"action": "call", "name": "qbonyekl.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE rating BETWEEN 5 AND 15;"}}, "output": "{\"action\":\"call\",\"name\":\"qbonyekl.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE rating BETWEEN 5 AND 15;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "94b5cc87d7b229fd2427_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "missing_schema", "question": "Maine database schema provide nahi kiya hai, isliye deliveries ke liye SQL likhne se pehle available tables find karo.", "expected": {"action": "call", "name": "dzrczeglaenew__list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"dzrczeglaenew__list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "e3314881b15c1b561692_en__development_layout", "language": "en", "backend": "supabase", "operation": "having", "question": "Identify the region groups in rentals with more than 10000 rows and report their respective counts.", "expected": {"action": "call", "name": "oqjfktjxcqkl_execute_sql", "arguments": {"query": "SELECT region, COUNT(*) FROM rentals GROUP BY region HAVING COUNT(*) > 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"oqjfktjxcqkl_execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM rentals GROUP BY region HAVING COUNT(*) > 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a7342b948d7bd031d82c_hinglish__development_layout", "language": "hinglish", "backend": "supabase", "operation": "min", "question": "deliveries mein cost ka minimum limit kya hai?", "expected": {"action": "call", "name": "kwwnskesvfmy_execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT MIN(cost) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"kwwnskesvfmy_execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT MIN(cost) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fe5044da918260a617a9_hinglish__development_layout", "language": "hinglish", "backend": "supabase", "operation": "group_sum", "question": "Mujhe repairs me har status ke liye cost ka total dikhana hai.", "expected": {"action": "call", "name": "qnyahklbq_execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT status, SUM(cost) FROM repairs GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"qnyahklbq_execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT status, SUM(cost) FROM repairs GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3d12ba589211e391881d_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "not_null", "question": "Show the repairs rows where title is not null.", "expected": {"action": "call", "name": "ejiejwdmpo_execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE title IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"ejiejwdmpo_execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE title IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e27bab397ec7daf2a101_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "distinct", "question": "enrollments में मौजूद डेटा से kind के विशिष्ट मान निकालें।", "expected": {"action": "call", "name": "lmojn__execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT DISTINCT kind FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"lmojn__execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT DISTINCT kind FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a5906e7b7aab0171e5db_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "avg", "question": "rentals me cost ka average kya hai?", "expected": {"action": "call", "name": "yuolnzw__execute_sql", "arguments": {"sql": "SELECT AVG(cost) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"yuolnzw__execute_sql\",\"arguments\":{\"sql\":\"SELECT AVG(cost) FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c752dd15f0913fe63436_en__development_layout", "language": "en", "backend": "supabase", "operation": "bottom", "question": "Pull the 5 rows with the lowest cost scores from repairs, smallest to largest.", "expected": {"action": "call", "name": "bpsvy.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs ORDER BY cost ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"bpsvy.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs ORDER BY cost ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1c48c9c4894db5982e9d_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "sql_error", "question": "You had a unknown column error in the last query. Check rentals schema before trying again.", "expected": {"action": "call", "name": "ajvobj__describe_table", "arguments": {"table": "rentals"}}, "output": "{\"action\":\"call\",\"name\":\"ajvobj__describe_table\",\"arguments\":{\"table\":\"rentals\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "32af759abb6742f3aa01_en__development_layout", "language": "en", "backend": "mysql", "operation": "all", "question": "Bring up all rows from enrollments.", "expected": {"action": "call", "name": "pkqhcdapthh.query", "arguments": {"query": "SELECT * FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"pkqhcdapthh.query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f0c710492ac60feed37b_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "group_count", "question": "enrollments में प्रत्येक region के पंक्ति काउंट दें।", "expected": {"action": "call", "name": "nhydloiqjro__execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT region, COUNT(*) FROM enrollments GROUP BY region;"}}, "output": "{\"action\":\"call\",\"name\":\"nhydloiqjro__execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT region, COUNT(*) FROM enrollments GROUP BY region;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8d65ec475048a7f3c9da_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "sort_asc", "question": "enrollments की सभी पंक्तियाँ salary के कम से ज्यादा क्रम में क्रमबद्ध करें।", "expected": {"action": "call", "name": "yaoehmhqaaj_execute_sql", "arguments": {"query": "SELECT * FROM enrollments ORDER BY salary ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"yaoehmhqaaj_execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments ORDER BY salary ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9ef4d4f5d5bcf5858087_en__development_layout", "language": "en", "backend": "supabase", "operation": "and", "question": "Select rows from repairs where city is Electronics and price is greater than 10.", "expected": {"action": "call", "name": "pzxcdmesmeoqv__execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE city = 'Electronics' AND price > 10;"}}, "output": "{\"action\":\"call\",\"name\":\"pzxcdmesmeoqv__execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE city = 'Electronics' AND price > 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9b4ac5b2bb0340639986_en__development_layout", "language": "en", "backend": "mysql", "operation": "month", "question": "Provide rows from enrollments where order_date's month value is 6.", "expected": {"action": "call", "name": "azneoub.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE MONTH(order_date) = 6;"}}, "output": "{\"action\":\"call\",\"name\":\"azneoub.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE MONTH(order_date) = 6;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "03695bef6508dec8ebdb_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "join_filter", "question": "select enrollments.customer_name from enrollments joined to enrollments_groups on enrollments.group_id where enrollments_groups.label is Delhi please", "expected": {"action": "call", "name": "ioesd.execute_sql", "arguments": {"query": "SELECT enrollments.customer_name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"ioesd.execute_sql\",\"arguments\":{\"query\":\"SELECT enrollments.customer_name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a888840d69a23410b604_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "top", "question": "repairs में quantity के उच्चतम मानों वाली 5 पंक्तियाँ प्रदर्शित करें।", "expected": {"action": "call", "name": "wwzbxn.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs ORDER BY quantity DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"wwzbxn.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs ORDER BY quantity DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fc9a19b2881b35db88b1_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "gt", "question": "मुझे deliveries से वे पंक्तियाँ चाहिए जहाँ score 500 से अधिक है।", "expected": {"action": "call", "name": "fgzcuhz_execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE score > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"fgzcuhz_execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE score > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "04165cd31bcf23c6fac9_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "eq", "question": "repairs se query karo aur wo saari rows dikhao jahan category O'Reilly hai.", "expected": {"action": "call", "name": "dhunag.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE category = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"dhunag.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE category = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f9c2698db9e7dbe6ec8e_hinglish__development_layout", "language": "hinglish", "backend": "supabase", "operation": "date_after", "question": "enrollments se wo records dikhao jahan start_date 2025-09-15 se pehle nahi hai.", "expected": {"action": "call", "name": "jcxqdngixynb_execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE start_date >= '2025-09-15';"}}, "output": "{\"action\":\"call\",\"name\":\"jcxqdngixynb_execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE start_date >= '2025-09-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5e4c3931848b0640bfaa_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "top", "question": "Show the top 10 entries in deliveries by quantity desc order.", "expected": {"action": "call", "name": "nipnsmn_execute_sql", "arguments": {"query": "SELECT * FROM deliveries ORDER BY quantity DESC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"nipnsmn_execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries ORDER BY quantity DESC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "46c2ec98a037d78a9576_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "list_tables", "question": "list out the tables in this database", "expected": {"action": "call", "name": "bljol.list_tables", "arguments": {"project_id": "demo_deliveries", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"bljol.list_tables\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "c3c868696c82e387acd8_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "gte", "question": "enrollments की वे एंट्री प्राप्त करें जहाँ balance 0 या उससे ऊपर है।", "expected": {"action": "call", "name": "zrnspyogdw.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE balance >= 0;"}}, "output": "{\"action\":\"call\",\"name\":\"zrnspyogdw.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE balance >= 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0e015b584f6a6179890c_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "date_after", "question": "rentals se wo records dikhao jahan created_at 2023-05-15 se pehle nahi hai.", "expected": {"action": "call", "name": "uwbsok_execute_sql", "arguments": {"sql": "SELECT * FROM rentals WHERE created_at >= '2023-05-15';"}}, "output": "{\"action\":\"call\",\"name\":\"uwbsok_execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE created_at >= '2023-05-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e6b1d053d0a307dba955_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "or", "question": "rentals से department Books या North वाली पंक्तियाँ लें।", "expected": {"action": "call", "name": "wozstpwpno__execute_sql", "arguments": {"project_id": "demo_rentals", "query": "SELECT * FROM rentals WHERE department IN ('Books', 'North');"}}, "output": "{\"action\":\"call\",\"name\":\"wozstpwpno__execute_sql\",\"arguments\":{\"project_id\":\"demo_rentals\",\"query\":\"SELECT * FROM rentals WHERE department IN ('Books', 'North');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dae8c9c830c197ee8e74_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "sort_desc", "question": "show all repairs rows ordered by salary highest to lowest", "expected": {"action": "call", "name": "khjkyma__query", "arguments": {"query": "SELECT * FROM repairs ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"khjkyma__query\",\"arguments\":{\"query\":\"SELECT * FROM repairs ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6d9c843a92d3ec7fb99c_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "sort_desc", "question": "show all rentals rows ordered by score highest to lowest", "expected": {"action": "call", "name": "bilqfezuqo_query", "arguments": {"query": "SELECT * FROM rentals ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"bilqfezuqo_query\",\"arguments\":{\"query\":\"SELECT * FROM rentals ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6391074b92a18a2d8964_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "project", "question": "bring up display_name from enrollments only", "expected": {"action": "call", "name": "iorxaxux_run_query", "arguments": {"query": "SELECT display_name FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"iorxaxux_run_query\",\"arguments\":{\"query\":\"SELECT display_name FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a89baf1622de706da189_en__development_layout", "language": "en", "backend": "mysql", "operation": "and", "question": "Select rows from rentals where status is Pune and amount is greater than 100.", "expected": {"action": "call", "name": "xnghzalfc.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE status = 'Pune' AND amount > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"xnghzalfc.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE status = 'Pune' AND amount > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "33d0ab3caa0fba0e3985_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "count", "question": "repairs ki rows ka volume kya hai?", "expected": {"action": "call", "name": "ffeqbkadu.run_query", "arguments": {"query": "SELECT COUNT(*) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"ffeqbkadu.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c4a83c9634139691ddde_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "having", "question": "repairs se 200 rows se zyada wale category groups ko unki row count ke saath dikhaiye.", "expected": {"action": "call", "name": "lmmwk_run_query", "arguments": {"sql": "SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 200;"}}, "output": "{\"action\":\"call\",\"name\":\"lmmwk_run_query\",\"arguments\":{\"sql\":\"SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "780db517b686cfd2fce3_en__development_layout", "language": "en", "backend": "supabase", "operation": "list_tables", "question": "Fetch the table list for this database.", "expected": {"action": "call", "name": "hgpadnnai_list_tables", "arguments": {"project_id": "demo_rentals", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"hgpadnnai_list_tables\",\"arguments\":{\"project_id\":\"demo_rentals\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a509f9b46620c564c0cf_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "missing_schema", "question": "मेने डेटाबेस स्कीमा प्रदान नहीं की है, इसलिए repairs के लिए SQL लिखने से पहले उपलब्ध तालिकाओं को खोजें।", "expected": {"action": "call", "name": "ezrtnfyote.list_tables", "arguments": {"project_id": "demo_repairs", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"ezrtnfyote.list_tables\",\"arguments\":{\"project_id\":\"demo_repairs\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "d8445da89ec6e3d5a191_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "distinct", "question": "rentals mein maujood data se category ke distinct values pull karo.", "expected": {"action": "call", "name": "clxnocvabc__execute_sql", "arguments": {"sql": "SELECT DISTINCT category FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"clxnocvabc__execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT category FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1667f375dfed0ad934a4_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "and", "question": "give me rows from repairs where city is Books and rating exceeds 1500", "expected": {"action": "call", "name": "lhyog_run_query", "arguments": {"query": "SELECT * FROM repairs WHERE city = 'Books' AND rating > 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"lhyog_run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE city = 'Books' AND rating > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4996e2672571480e6c60_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "project", "question": "i want just customer_name from rentals ok", "expected": {"action": "call", "name": "fwdokqtuar__run_query", "arguments": {"sql": "SELECT customer_name FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"fwdokqtuar__run_query\",\"arguments\":{\"sql\":\"SELECT customer_name FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1c48c9c4894db5982e9d_en__development_layout", "language": "en", "backend": "mysql", "operation": "sql_error", "question": "Please review the structure of rentals before retrying, as the last query failed on an unrecognized column.", "expected": {"action": "call", "name": "zbffrrc.describe_table", "arguments": {"table": "rentals"}}, "output": "{\"action\":\"call\",\"name\":\"zbffrrc.describe_table\",\"arguments\":{\"table\":\"reviews\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "8c437f27ac8f48915f9a_hi__development_layout", "language": "hi", "backend": "mysql", "operation": "min", "question": "rentals में balance का न्यूनतम स्तर क्या है?", "expected": {"action": "call", "name": "bsthyan__run_query", "arguments": {"sql": "SELECT MIN(balance) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"bsthyan__run_query\",\"arguments\":{\"sql\":\"SELECT MIN(balance) FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "467e53c90a42cd9935a3_en__development_layout", "language": "en", "backend": "mysql", "operation": "lte", "question": "Show the rentals data where price is at most 5.", "expected": {"action": "call", "name": "hwevpuaz_run_query", "arguments": {"query": "SELECT * FROM rentals WHERE price <= 5;"}}, "output": "{\"action\":\"call\",\"name\":\"hwevpuaz_run_query\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE price <= 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2fae606ab30f319280e2_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "month", "question": "rentals mein order_date ka month 12 hone wali rows dikhao.", "expected": {"action": "call", "name": "khvdfpskbj__query", "arguments": {"query": "SELECT * FROM rentals WHERE MONTH(order_date) = 12;"}}, "output": "{\"action\":\"call\",\"name\":\"khvdfpskbj__query\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE MONTH(order_date) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "928e148fdcacd00b97f8_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "sql_error", "question": "You had a unknown column error in the last query. Check deliveries schema before trying again.", "expected": {"action": "call", "name": "zeaftbb_describe_table", "arguments": {"table": "deliveries"}}, "output": "{\"action\":\"call\",\"name\":\"zeaftbb_describe_table\",\"arguments\":{\"table\":\"deliveries\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "c7aa166bcab44f298306_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "top", "question": "Show the top 1 entries in enrollments by cost desc order.", "expected": {"action": "call", "name": "hiqpxjcexiwha.query", "arguments": {"sql": "SELECT * FROM enrollments ORDER BY cost DESC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"hiqpxjcexiwha.query\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments ORDER BY cost DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "660dd90727fa904c6f30_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "top", "question": "rentals mein cost ke highest values wali 10 rows display karo.", "expected": {"action": "call", "name": "hcyyame_execute_sql", "arguments": {"sql": "SELECT * FROM rentals ORDER BY cost DESC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"hcyyame_execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM rentals ORDER BY cost DESC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6aaf184e4cafcf2f5c16_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "having", "question": "Display category groups in repairs where row count is > 20 and show the counts.", "expected": {"action": "call", "name": "cdmdlyl.query", "arguments": {"query": "SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"cdmdlyl.query\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f12c053fdb90a634ce20_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "sql_error", "question": "Kyunki aakhri query unknown column ki wajah se fail hui, repairs schema inspect karne ke baad hi dobara try karo.", "expected": {"action": "call", "name": "owihiereyg_describe_table", "arguments": {"table": "repairs"}}, "output": "{\"action\":\"call\",\"name\":\"owihiereyg_describe_table\",\"arguments\":{\"table\":\"repairs\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a2072435473a505c6fff_hi__development_layout", "language": "hi", "backend": "mysql", "operation": "all", "question": "repairs से सभी पंक्तियाँ लाइए।", "expected": {"action": "call", "name": "narpglwmbc__run_query", "arguments": {"query": "SELECT * FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"narpglwmbc__run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE city = 'Agairs';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "no such column: city"}}
{"id": "dae8c9c830c197ee8e74_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "sort_desc", "question": "repairs ke saare rows salary highest se lowest order mein return karo.", "expected": {"action": "call", "name": "nkztwcm__query", "arguments": {"query": "SELECT * FROM repairs ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"nkztwcm__query\",\"arguments\":{\"query\":\"SELECT * FROM repairs ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "46c2ec98a037d78a9576_hinglish__development_layout", "language": "hinglish", "backend": "supabase", "operation": "list_tables", "question": "Kripya is database ki tables fetch karo.", "expected": {"action": "call", "name": "rrosbgyyo.list_tables", "arguments": {"project_id": "demo_deliveries", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"rrosbgyyo.list_tables\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "78e1c795f40293f04ae5_en__development_layout", "language": "en", "backend": "mysql", "operation": "sum", "question": "Measure the total quantity present in deliveries.", "expected": {"action": "call", "name": "dwxoz_execute_sql", "arguments": {"sql": "SELECT SUM(quantity) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"dwxoz_execute_sql\",\"arguments\":{\"sql\":\"SELECT SUM(quantity) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3f2bcb817dc0ac7fb510_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "sort_asc", "question": "retrieve all rows from enrollments and put them in order by score lowest first", "expected": {"action": "call", "name": "tqiykjigkvw_execute_sql", "arguments": {"query": "SELECT * FROM enrollments ORDER BY score ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"tqiykjigkvw_execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments ORDER BY score ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bff6b3ccadd4c292165c_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "sql_error", "question": "You had a unknown column error in the last query. Check deliveries schema before trying again.", "expected": {"action": "call", "name": "mjcckozareci__describe_table", "arguments": {"table": "deliveries"}}, "output": "{\"action\":\"call\",\"name\":\"mjcckozareci__describe_table\",\"arguments\":{\"table\":\"deliveries\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "494e7ce9785848f5299f_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "describe", "question": "Display the field types and names for rentals.", "expected": {"action": "call", "name": "tfqzzsprny.list_tables", "arguments": {"project_id": "demo_rentals", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"tfqzzsprny.list_tables\",\"arguments\":{\"project_id\":\"demo_rentals\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "0e015b584f6a6179890c_en__development_layout", "language": "en", "backend": "mysql", "operation": "date_after", "question": "Query rentals for rows where created_at is at or after 2023-05-15.", "expected": {"action": "call", "name": "bkkyubixx__execute_sql", "arguments": {"sql": "SELECT * FROM rentals WHERE created_at >= '2023-05-15';"}}, "output": "{\"action\":\"call\",\"name\":\"bkkyubixx__execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE created_at >= '2023-05-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f661788465b66e8a5822_en__development_layout", "language": "en", "backend": "mysql", "operation": "lt", "question": "Present the data from enrollments where score is below 200.", "expected": {"action": "call", "name": "qfryvqlotwx__execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE score < 200;"}}, "output": "{\"action\":\"call\",\"name\":\"qfryvqlotwx__execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE score < 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "95a0b0a0ed8d648db327_hinglish__development_layout", "language": "hinglish", "backend": "supabase", "operation": "join_filter", "question": "Un rows ka enrollments.name lao jahan enrollments.group_id enrollments_groups.id se juda hai aur enrollments_groups.label South ke barabar hai.", "expected": {"action": "call", "name": "xszblbkrkxz_execute_sql", "arguments": {"query": "SELECT enrollments.name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"xszblbkrkxz_execute_sql\",\"arguments\":{\"query\":\"SELECT enrollments.name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9325b9d4f4ee1c176f2f_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "gt", "question": "look up the rows in repairs where score is greater than 1500", "expected": {"action": "call", "name": "qcxqe__execute_sql", "arguments": {"sql": "SELECT * FROM repairs WHERE score > 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"qcxqe__execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM repairs WHERE score > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0e9353d09ce2f0cd9994_en__development_layout", "language": "en", "backend": "mysql", "operation": "count_eq", "question": "How many instances in deliveries have city as O'Reilly?", "expected": {"action": "call", "name": "dildvxmoe.execute_sql", "arguments": {"sql": "SELECT COUNT(*) FROM deliveries WHERE city = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"dildvxmoe.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM deliveries WHERE city = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3920dafebc98270ffed5_en__development_layout", "language": "en", "backend": "mysql", "operation": "between", "question": "Output the rows from enrollments where score is >= 500 and <= 1000.", "expected": {"action": "call", "name": "bykxnoxxbj_query", "arguments": {"query": "SELECT * FROM enrollments WHERE score BETWEEN 500 AND 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"bykxnoxxbj_query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE score BETWEEN 500 AND 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c3d3a3d5475ef70e2f2c_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "sort_asc", "question": "enrollments ki saari rows price ke kam se zyada order mein sort karo.", "expected": {"action": "call", "name": "lqnmn__query", "arguments": {"sql": "SELECT * FROM enrollments ORDER BY price ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"lqnmn__query\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments ORDER BY price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2d72fda52b91ee1560ab_en__development_layout", "language": "en", "backend": "mysql", "operation": "all", "question": "Bring up all rows from deliveries.", "expected": {"action": "call", "name": "ngjjdskqnuept.run_query", "arguments": {"sql": "SELECT * FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"ngjjdskqnuept.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "00e1fcbe466a7fdf0d8a_en__development_layout", "language": "en", "backend": "mysql", "operation": "date_after", "question": "Query rentals for rows where order_date is at or after 2023-11-15.", "expected": {"action": "call", "name": "qqiyx__run_query", "arguments": {"sql": "SELECT * FROM rentals WHERE order_date >= '2023-11-15';"}}, "output": "{\"action\":\"call\",\"name\":\"qqiyx__run_query\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE order_date >= '2023-11-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bff6b3ccadd4c292165c_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "sql_error", "question": "Kyunki aakhri query unknown column ki wajah se fail hui, deliveries schema inspect karne ke baad hi dobara try karo.", "expected": {"action": "call", "name": "ishalbsryqqj_describe_table", "arguments": {"table": "deliveries"}}, "output": "{\"action\":\"call\",\"name\":\"ishalbsryqqj_describe_table\",\"arguments\":{\"table\":\"deliveries\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "2a70571066edacbb19ac_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "bottom", "question": "rentals से score के आधार पर 5 न्यूनतम पंक्तियां, सबसे कम मान से शुरू होकर, प्रदर्शित करें।", "expected": {"action": "call", "name": "iytxtqt__execute_sql", "arguments": {"project_id": "demo_rentals", "query": "SELECT * FROM rentals ORDER BY score ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"iytxtqt__execute_sql\",\"arguments\":{\"project_id\":\"demo_rentals\",\"query\":\"SELECT * FROM rentals ORDER BY score ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8bacbbfb4da1879a2d23_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "list_tables", "question": "Kripya is database ki tables fetch karo.", "expected": {"action": "call", "name": "actharla.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"actharla.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "b6da31e0b488a9ecb9e4_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "count_eq", "question": "help me count rows in rentals where region is Books", "expected": {"action": "call", "name": "woosrmqq__query", "arguments": {"sql": "SELECT COUNT(*) FROM rentals WHERE region = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"woosrmqq__query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM rentals WHERE region = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a2072435473a505c6fff_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "all", "question": "pull all rows from repairs", "expected": {"action": "call", "name": "cqgrwpgfhkix_run_query", "arguments": {"query": "SELECT * FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"cqgrwpgfhkix_run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fc1a5aaf55bd30034ad0_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "count", "question": "deliveries ki rows ka volume kya hai?", "expected": {"action": "call", "name": "baujnhilijgvl.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"baujnhilijgvl.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f12c053fdb90a634ce20_en__development_layout", "language": "en", "backend": "mysql", "operation": "sql_error", "question": "Please review the structure of repairs before retrying, as the last query failed on an unrecognized column.", "expected": {"action": "call", "name": "ovoirzsdhjh.describe_table", "arguments": {"table": "repairs"}}, "output": "{\"action\":\"call\",\"name\":\"ovoirzsdhjh.describe_table\",\"arguments\":{\"table\":\"repairs\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "2d039adc19e8ba87f706_hi__development_layout", "language": "hi", "backend": "mysql", "operation": "or", "question": "deliveries से category Pune या North वाली पंक्तियाँ लें।", "expected": {"action": "call", "name": "kzhwj_run_query", "arguments": {"query": "SELECT * FROM deliveries WHERE category IN ('Pune', 'North');"}}, "output": "{\"action\":\"call\",\"name\":\"kzhwj_run_query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE category IN ('Pune', 'North');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4996e2672571480e6c60_hi__development_layout", "language": "hi", "backend": "mysql", "operation": "project", "question": "मुझे rentals का customer_name चाहिए, बाकी कुछ नहीं।", "expected": {"action": "call", "name": "kegzenszr.run_query", "arguments": {"sql": "SELECT customer_name FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"kegzenszr.run_query\",\"arguments\":{\"sql\":\"SELECT customer_name FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3920dafebc98270ffed5_hi__development_layout", "language": "hi", "backend": "mysql", "operation": "between", "question": "enrollments से वे पंक्तियाँ प्रदर्शित करें जहाँ score >= 500 और <= 1000 है।", "expected": {"action": "call", "name": "gwjbwc.query", "arguments": {"query": "SELECT * FROM enrollments WHERE score BETWEEN 500 AND 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"gwjbwc.query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE score BETWEEN 500 AND 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "00e1fcbe466a7fdf0d8a_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "date_after", "question": "Show me rentals data where order_date is not earlier than 2023-11-15.", "expected": {"action": "call", "name": "rvgtgcmgby_run_query", "arguments": {"sql": "SELECT * FROM rentals WHERE order_date >= '2023-11-15';"}}, "output": "{\"action\":\"call\",\"name\":\"rvgtgcmgby_run_query\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE order_date >= '2023-11-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b654c9f6e4c53883151c_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "join", "question": "retrieve repairs.description and repairs_groups.label join repairs.group_id to repairs_groups.id", "expected": {"action": "call", "name": "kqsca_execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT repairs.description, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"kqsca_execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT repairs.description, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b8b9810be462344512fe_hi__development_layout", "language": "hi", "backend": "mysql", "operation": "min", "question": "deliveries में score का न्यूनतम स्तर क्या है?", "expected": {"action": "call", "name": "ljwbhldpxf.execute_sql", "arguments": {"query": "SELECT MIN(score) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"ljwbhldpxf.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(score) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "21498b30adb6d60f9630_en__development_layout", "language": "en", "backend": "mysql", "operation": "min", "question": "Give me the minimum amount from enrollments.", "expected": {"action": "call", "name": "leoqrixidqp__run_query", "arguments": {"query": "SELECT MIN(amount) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"leoqrixidqp__run_query\",\"arguments\":{\"query\":\"SELECT MIN(amount) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8bacbbfb4da1879a2d23_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "list_tables", "question": "list out the tables in this database", "expected": {"action": "call", "name": "nzykkym_list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"nzykkym_list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "b13605951ead6efc54ba_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "describe", "question": "Display the field types and names for rentals.", "expected": {"action": "call", "name": "rgosgqa.describe_table", "arguments": {"table": "rentals"}}, "output": "{\"action\":\"call\",\"name\":\"rgosgqa.describe_table\",\"arguments\":{\"table\":\"rentals\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "c147e867055d7fd9f259_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "list_tables", "question": "Kripya is database ki tables fetch karo.", "expected": {"action": "call", "name": "ginzcpmd_list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"ginzcpmd_list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "84a09238c3eb1043c14c_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "avg", "question": "give me avg salary in repairs", "expected": {"action": "call", "name": "vgmjjzvjop_query", "arguments": {"sql": "SELECT AVG(salary) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"vgmjjzvjop_query\",\"arguments\":{\"sql\":\"SELECT AVG(salary) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dbc778be5f23038bcc3d_en__development_layout", "language": "en", "backend": "mysql", "operation": "count_eq", "question": "How many instances in repairs have category as O'Reilly?", "expected": {"action": "call", "name": "osvuyzpzgm__run_query", "arguments": {"query": "SELECT COUNT(*) FROM repairs WHERE category = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"osvuyzpzgm__run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM repairs WHERE category = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "78e1c795f40293f04ae5_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "sum", "question": "please find the total quantity in deliveries", "expected": {"action": "call", "name": "qwpvnxxcrmuc_execute_sql", "arguments": {"sql": "SELECT SUM(quantity) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"qwpvnxxcrmuc_execute_sql\",\"arguments\":{\"sql\":\"SELECT SUM(quantity) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3920dafebc98270ffed5_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "between", "question": "enrollments se wo rows show karo jahan score >= 500 aur <= 1000 hai.", "expected": {"action": "call", "name": "nygfe_query", "arguments": {"query": "SELECT * FROM enrollments WHERE score BETWEEN 500 AND 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"nygfe_query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE score BETWEEN 500 AND 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5e4c3931848b0640bfaa_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "top", "question": "deliveries mein quantity ke highest values wali 10 rows display karo.", "expected": {"action": "call", "name": "dqpgwj__execute_sql", "arguments": {"query": "SELECT * FROM deliveries ORDER BY quantity DESC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"dqpgwj__execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries ORDER BY quantity DESC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "660dd90727fa904c6f30_hi__development_layout", "language": "hi", "backend": "mysql", "operation": "top", "question": "rentals में cost के उच्चतम मानों वाली 10 पंक्तियाँ प्रदर्शित करें।", "expected": {"action": "call", "name": "cvrjkhvyfo_execute_sql", "arguments": {"sql": "SELECT * FROM rentals ORDER BY cost DESC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"cvrjkhvyfo_execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM rentals ORDER BY cost DESC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b9d861b71317ad86ce3e_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "having", "question": "Display category groups in enrollments where row count is > 2 and show the counts.", "expected": {"action": "call", "name": "sauxphycgf_execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM enrollments GROUP BY category HAVING COUNT(*) > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"sauxphycgf_execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM enrollments GROUP BY category HAVING COUNT(*) > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "68d0c440e1210d5f5590_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "group_sum", "question": "Mujhe repairs me har kind ke liye score ka total dikhana hai.", "expected": {"action": "call", "name": "brqnxgu__query", "arguments": {"sql": "SELECT kind, SUM(score) FROM repairs GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"brqnxgu__query\",\"arguments\":{\"sql\":\"SELECT kind, SUM(score) FROM repairs GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8aeca0843f974e95bd97_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "list_tables", "question": "list out the tables in this database", "expected": {"action": "call", "name": "zgapcaj.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"zgapcaj.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "46c2ec98a037d78a9576_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "list_tables", "question": "कृपया इस डेटाबेस की तालिकाएँ fetch करें।", "expected": {"action": "call", "name": "ucyjzslvirao_list_tables", "arguments": {"project_id": "demo_deliveries", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"ucyjzslvirao_list_tables\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "84a09238c3eb1043c14c_hi__development_layout", "language": "hi", "backend": "mysql", "operation": "avg", "question": "repairs में salary का औसत क्या है?", "expected": {"action": "call", "name": "slwucagfcq.query", "arguments": {"sql": "SELECT AVG(salary) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"slwucagfcq.query\",\"arguments\":{\"sql\":\"SELECT AVG(salary) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6230fff878aa525542cd_hi__development_layout", "language": "hi", "backend": "mysql", "operation": "missing_schema", "question": "मेने डेटाबेस स्कीमा प्रदान नहीं की है, इसलिए enrollments के लिए SQL लिखने से पहले उपलब्ध तालिकाओं को खोजें।", "expected": {"action": "call", "name": "nmjdgjglp.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"nmjdgjglp.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "5cf994b39a1eb0a5d0b6_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "null", "question": "deliveries mein item_name null hone wali rows lao.", "expected": {"action": "call", "name": "zwemgwwgieko__execute_sql", "arguments": {"sql": "SELECT * FROM deliveries WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"zwemgwwgieko__execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b93dfa01b85949af4005_hi__development_layout", "language": "hi", "backend": "mysql", "operation": "contains", "question": "deliveries में description में Mumbai रखने वाली पंक्तियाँ सूचीबद्ध करें, मामला-असंवेदनशीलता के साथ।", "expected": {"action": "call", "name": "qjvcui_query", "arguments": {"query": "SELECT * FROM deliveries WHERE LOWER(description) LIKE LOWER('%Mumbai%');"}}, "output": "{\"action\":\"call\",\"name\":\"qjvcui_query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE LOWER(description) LIKE LOWER('%Mumbai%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "32989df4383378821b6c_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "describe", "question": "Display the columns and data types for deliveries.", "expected": {"action": "call", "name": "mpwkupmydr_describe_table", "arguments": {"table": "deliveries"}}, "output": "{\"action\":\"call\",\"name\":\"mpwkupmydr_describe_table\",\"arguments\":{\"table\":\"deliveries\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "5e4c3931848b0640bfaa_hi__development_layout", "language": "hi", "backend": "mysql", "operation": "top", "question": "deliveries में quantity के उच्चतम मानों वाली 10 पंक्तियाँ प्रदर्शित करें।", "expected": {"action": "call", "name": "ewuoa_execute_sql", "arguments": {"query": "SELECT * FROM deliveries ORDER BY quantity DESC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"ewuoa_execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries ORDER BY quantity DESC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "af4683485d1360b8afb9_hi__development_layout", "language": "hi", "backend": "mysql", "operation": "max", "question": "repairs में quantity की अधिकतम संख्या क्या है?", "expected": {"action": "call", "name": "cvqcxmisswzhw_query", "arguments": {"query": "SELECT MAX(quantity) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"cvqcxmisswzhw_query\",\"arguments\":{\"query\":\"SELECT MAX(quantity) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c147e867055d7fd9f259_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "list_tables", "question": "list out the tables in this database", "expected": {"action": "call", "name": "rhpoqnfvovvc.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"rhpoqnfvovvc.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "f637ccd41642ab190d04_en__development_layout", "language": "en", "backend": "mysql", "operation": "null", "question": "Show me the rows from repairs where customer_name is null.", "expected": {"action": "call", "name": "hlmqvbzgoydr__run_query", "arguments": {"query": "SELECT * FROM repairs WHERE customer_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"hlmqvbzgoydr__run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE customer_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4a26377ced6478bb8d0e_hi__development_layout", "language": "hi", "backend": "mysql", "operation": "and", "question": "repairs से वे पंक्तियाँ प्रदर्शित करें जहाँ city Electronics है और quantity 200 से अधिक है।", "expected": {"action": "call", "name": "komxxgxqdk__query", "arguments": {"sql": "SELECT * FROM repairs WHERE city = 'Electronics' AND quantity > 200;"}}, "output": "{\"action\":\"call\",\"name\":\"komxxgxqdk__query\",\"arguments\":{\"sql\":\"SELECT * FROM repairs WHERE city = 'Electronics' AND quantity > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "32989df4383378821b6c_hi__development_layout", "language": "hi", "backend": "mysql", "operation": "describe", "question": "deliveries में मौजूद फील्ड्स और टाइप्स बताएं।", "expected": {"action": "call", "name": "ademstmoa_describe_table", "arguments": {"table": "deliveries"}}, "output": "{\"action\":\"call\",\"name\":\"ademstmoa_describe_table\",\"arguments\":{\"table\":\"deliveries\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "c147e867055d7fd9f259_en__development_layout", "language": "en", "backend": "mysql", "operation": "list_tables", "question": "Fetch the table list for this database.", "expected": {"action": "call", "name": "sdgpbrjxdvlwz__list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"sdgpbrjxdvlwz__list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "b654c9f6e4c53883151c_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "join", "question": "repairs.group_id को repairs_groups.id से जोड़कर repairs.description और repairs_groups.label का डेटा दिखाएँ।", "expected": {"action": "call", "name": "cfyfnl.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT repairs.description, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"cfyfnl.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT repairs.description, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1339ee90b3d8b8eac1f7_en__development_layout", "language": "en", "backend": "mysql", "operation": "join_filter", "question": "Select enrollments.name from enrollments joined with enrollments_groups on enrollments.group_id where enrollments_groups.label is North.", "expected": {"action": "call", "name": "wsoxnieeix__run_query", "arguments": {"sql": "SELECT enrollments.name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"wsoxnieeix__run_query\",\"arguments\":{\"sql\":\"SELECT enrollments.name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8bacbbfb4da1879a2d23_hi__development_layout", "language": "hi", "backend": "mysql", "operation": "list_tables", "question": "कृपया इस डेटाबेस की तालिकाएँ fetch करें।", "expected": {"action": "call", "name": "hudqkcsjlpo__list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"hudqkcsjlpo__list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "7cf5c995d18de3fe3679_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "join", "question": "retrieve repairs.display_name and repairs_groups.label join repairs.group_id to repairs_groups.id", "expected": {"action": "call", "name": "kzrjte.run_query", "arguments": {"sql": "SELECT repairs.display_name, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"kzrjte.run_query\",\"arguments\":{\"sql\":\"SELECT repairs.display_name, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bff6b3ccadd4c292165c_hi__development_layout", "language": "hi", "backend": "mysql", "operation": "sql_error", "question": "क्योंकि अंतिम क्वेरी अज्ञात कॉलम के कारण फेल हुई, deliveries स्कीमा का निरीक्षण करने के बाद ही दोबारा प्रयास करें।", "expected": {"action": "call", "name": "ydlhdch.describe_table", "arguments": {"table": "deliveries"}}, "output": "{\"action\":\"call\",\"name\":\"ydlhdch.describe_table\",\"arguments\":{\"table\":\"deliveries\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "8aeca0843f974e95bd97_hi__development_layout", "language": "hi", "backend": "mysql", "operation": "list_tables", "question": "कृपया इस डेटाबेस की तालिकाएँ fetch करें।", "expected": {"action": "call", "name": "awsepnrwqm.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"awsepnrwqm.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "efee860c507a44ecafbb_hi__development_layout", "language": "hi", "backend": "mysql", "operation": "sql_error", "question": "क्योंकि अंतिम क्वेरी अज्ञात कॉलम के कारण फेल हुई, deliveries स्कीमा का निरीक्षण करने के बाद ही दोबारा प्रयास करें।", "expected": {"action": "call", "name": "esagrggfujyq.describe_table", "arguments": {"table": "deliveries"}}, "output": "{\"action\":\"call\",\"name\":\"esagrggfujyq.describe_table\",\"arguments\":{\"table\":\"deliveries\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6575b299f08c87b34390_hinglish__development_layout", "language": "hinglish", "backend": "supabase", "operation": "search", "question": "Docs ke andar indexes dhoondo.", "expected": {"action": "call", "name": "oudzi_search_docs", "arguments": {"query": "indexes"}}, "output": "{\"action\":\"call\",\"name\":\"oudzi_search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "e416c627b8dd4b3c2bfc_hinglish__development_layout", "language": "hinglish", "backend": "supabase", "operation": "ticket", "question": "ticket TKT-9638 ki info retrieve karo.", "expected": {"action": "call", "name": "slwwpd_get_ticket", "arguments": {"ticket_id": "TKT-9638"}}, "output": "{\"action\":\"call\",\"name\":\"slwwpd_get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9638\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6575b299f08c87b34390_en__development_layout", "language": "en", "backend": "supabase", "operation": "search", "question": "Look for indexes inside the documentation.", "expected": {"action": "call", "name": "sezgtnggx.search_docs", "arguments": {"query": "indexes"}}, "output": "{\"action\":\"call\",\"name\":\"sezgtnggx.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "dd1576137b2d7927bedc_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "ticket", "question": "टिकट TKT-286 का डेटा फेच करें।", "expected": {"action": "call", "name": "kbffpxj_get_ticket", "arguments": {"ticket_id": "TKT-286"}}, "output": "{\"action\":\"call\",\"name\":\"kbffpxj_get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-286\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "dd1576137b2d7927bedc_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "ticket", "question": "display the record for TKT-286", "expected": {"action": "call", "name": "qutopipvqqd.get_ticket", "arguments": {"ticket_id": "TKT-286"}}, "output": "{\"action\":\"call\",\"name\":\"qutopipvqqd.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-286\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a150b9436cc95f4db00f_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "weather", "question": "Please show me weather for Delhi in fahrenheit.", "expected": {"action": "call", "name": "ofpwuevqgie_get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"ofpwuevqgie_get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a74cd18fb14dca57100e_en__development_layout", "language": "en", "backend": "supabase", "operation": "read_file", "question": "Load the content of the file at /docs/README.md.", "expected": {"action": "call", "name": "tympvnsgtsemy_read_file", "arguments": {"path": "/docs/README.md"}}, "output": "{\"action\":\"call\",\"name\":\"tympvnsgtsemy_read_file\",\"arguments\":{\"path\":\"/docs/README.md\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "dd1576137b2d7927bedc_en__development_layout", "language": "en", "backend": "supabase", "operation": "ticket", "question": "Please locate the information for support ticket TKT-286.", "expected": {"action": "call", "name": "nmxrjxvyi__get_ticket", "arguments": {"ticket_id": "TKT-286"}}, "output": "{\"action\":\"call\",\"name\":\"nmxrjxvyi__get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-286\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "e815e61cd1cbba2d7eec_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "read_file", "question": "Load the file at /reports/sales.csv and read it.", "expected": {"action": "call", "name": "guhyk_read_file", "arguments": {"path": "/reports/sales.csv"}}, "output": "{\"action\":\"call\",\"name\":\"guhyk_read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "9c2fcd765bf5c2df33aa_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "search", "question": "help me find row level security in the documentation", "expected": {"action": "call", "name": "smzdrr__search_docs", "arguments": {"query": "row level security"}}, "output": "{\"action\":\"call\",\"name\":\"smzdrr__search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a150b9436cc95f4db00f_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "weather", "question": "Delhi का मौसम fahrenheit में लाएं।", "expected": {"action": "call", "name": "kjvxyhndkbdl__get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"kjvxyhndkbdl__get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "2938bd358ea4809d346f_en__development_layout", "language": "en", "backend": "mysql", "operation": "search", "question": "Look for database backups inside the documentation.", "expected": {"action": "call", "name": "urvwv_search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"urvwv_search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "5e59d3140947b3c7bc4a_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "weather", "question": "Jaipur ka mausam fahrenheit mein lao.", "expected": {"action": "call", "name": "ziixva__get_weather", "arguments": {"city": "Jaipur", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"ziixva__get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "32d9828b8b0cba285197_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "ticket", "question": "display the record for TKT-7867", "expected": {"action": "call", "name": "hamtsodzkdouw__get_ticket", "arguments": {"ticket_id": "TKT-7867"}}, "output": "{\"action\":\"call\",\"name\":\"hamtsodzkdouw__get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-7867\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "48a316f4c32067e3ca40_hinglish__development_layout", "language": "hinglish", "backend": "supabase", "operation": "search", "question": "Docs ke andar row level security dhoondo.", "expected": {"action": "call", "name": "cgsquddlodmck_search_docs", "arguments": {"query": "row level security"}}, "output": "{\"action\":\"call\",\"name\":\"cgsquddlodmck_search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a9cd5b7c451419ebb9c3_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "read_file", "question": "/reports/sales.csv पर स्थित फ़ाइल का सामग्री लोड करें।", "expected": {"action": "call", "name": "aavnpo__read_file", "arguments": {"path": "/reports/sales.csv"}}, "output": "{\"action\":\"call\",\"name\":\"aavnpo__read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a74cd18fb14dca57100e_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "read_file", "question": "/docs/README.md पर स्थित फ़ाइल का सामग्री लोड करें।", "expected": {"action": "call", "name": "dgazrdwewzjk_read_file", "arguments": {"path": "/docs/README.md"}}, "output": "{\"action\":\"call\",\"name\":\"dgazrdwewzjk_read_file\",\"arguments\":{\"path\":\"/docs/README.md\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "c226064697cb1dc20646_en__development_layout", "language": "en", "backend": "mysql", "operation": "weather", "question": "Display the weather for Lucknow using celsius.", "expected": {"action": "call", "name": "ruxwxw_get_weather", "arguments": {"city": "Lucknow", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"ruxwxw_get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a9cd5b7c451419ebb9c3_en__development_layout", "language": "en", "backend": "supabase", "operation": "read_file", "question": "Load the content of the file at /reports/sales.csv.", "expected": {"action": "call", "name": "jxnlhzsiop_read_file", "arguments": {"path": "/reports/sales.csv"}}, "output": "{\"action\":\"call\",\"name\":\"jxnlhzsiop_read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "c226064697cb1dc20646_hi__development_layout", "language": "hi", "backend": "mysql", "operation": "weather", "question": "Lucknow का मौसम celsius में लाएं।", "expected": {"action": "call", "name": "wryrxsoir_get_weather", "arguments": {"city": "Lucknow", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"wryrxsoir_get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "132914c1610a3ceada4c_en__development_layout", "language": "en", "backend": "mysql", "operation": "ticket", "question": "Please locate the information for support ticket TKT-8718.", "expected": {"action": "call", "name": "huduaymyi_get_ticket", "arguments": {"ticket_id": "TKT-8718"}}, "output": "{\"action\":\"call\",\"name\":\"huduaymyi_get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-8718\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "32d9828b8b0cba285197_hi__development_layout", "language": "hi", "backend": "supabase", "operation": "ticket", "question": "टिकट TKT-7867 का डेटा फेच करें।", "expected": {"action": "call", "name": "wfqigjebnhg_get_ticket", "arguments": {"ticket_id": "TKT-7867"}}, "output": "{\"action\":\"call\",\"name\":\"wfqigjebnhg_get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-7867\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "5a26587ff6051e365791_hi__development_layout", "language": "hi", "backend": "mysql", "operation": "read_file", "question": "/notes/setup.txt पर स्थित फ़ाइल का सामग्री लोड करें।", "expected": {"action": "call", "name": "kldtdso_read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"kldtdso_read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "37a4acee442f2b6667ce_en__development_layout", "language": "en", "backend": "mysql", "operation": "search", "question": "Look for API authentication inside the documentation.", "expected": {"action": "call", "name": "oneusagvgjaeh.search_docs", "arguments": {"query": "API authentication"}}, "output": "{\"action\":\"call\",\"name\":\"oneusagvgjaeh.search_docs\",\"arguments\":{\"query\":\"API authentication\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "37a4acee442f2b6667ce_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "search", "question": "Docs ke andar API authentication dhoondo.", "expected": {"action": "call", "name": "ahzunegdhuf.search_docs", "arguments": {"query": "API authentication"}}, "output": "{\"action\":\"call\",\"name\":\"ahzunegdhuf.search_docs\",\"arguments\":{\"query\":\"API authentication\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "132914c1610a3ceada4c_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "ticket", "question": "ticket TKT-8718 ki info retrieve karo.", "expected": {"action": "call", "name": "kjasrmpsokz_get_ticket", "arguments": {"ticket_id": "TKT-8718"}}, "output": "{\"action\":\"call\",\"name\":\"kjasrmpsokz_get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-8718\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "132914c1610a3ceada4c_hi__development_layout", "language": "hi", "backend": "mysql", "operation": "ticket", "question": "टिकट TKT-8718 का डेटा फेच करें।", "expected": {"action": "call", "name": "bpnwklr.get_ticket", "arguments": {"ticket_id": "TKT-8718"}}, "output": "{\"action\":\"call\",\"name\":\"bpnwklr.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-8718\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a9cd5b7c451419ebb9c3_noisy_en__development_layout", "language": "noisy_en", "backend": "supabase", "operation": "read_file", "question": "Load the file at /reports/sales.csv and read it.", "expected": {"action": "call", "name": "tdtrnrwxcbt__read_file", "arguments": {"path": "/reports/sales.csv"}}, "output": "{\"action\":\"call\",\"name\":\"tdtrnrwxcbt__read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "b1f06f7bdfe02dd430e7_hi__development_layout", "language": "hi", "backend": "mysql", "operation": "weather", "question": "Delhi का मौसम fahrenheit में लाएं।", "expected": {"action": "call", "name": "wxmlbtinqhco_get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"wxmlbtinqhco_get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "63e80f5e293fa639c0d9_noisy_en__development_layout", "language": "noisy_en", "backend": "mysql", "operation": "weather", "question": "Please show me weather for Mumbai in fahrenheit.", "expected": {"action": "call", "name": "xszspziypy.get_weather", "arguments": {"city": "Mumbai", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"xszspziypy.get_weather\",\"arguments\":{\"city\":\"Mumbai\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "2938bd358ea4809d346f_hi__development_layout", "language": "hi", "backend": "mysql", "operation": "search", "question": "मुझे दस्तावेज़ीकरण में database backups की आवश्यकता है।", "expected": {"action": "call", "name": "sdkkxceyo__search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"sdkkxceyo__search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "eee82f0e2886d3c03463_hinglish__development_layout", "language": "hinglish", "backend": "mysql", "operation": "search", "question": "Docs ke andar row level security dhoondo.", "expected": {"action": "call", "name": "lzkmkvuijtbr.search_docs", "arguments": {"query": "row level security"}}, "output": "{\"action\":\"call\",\"name\":\"lzkmkvuijtbr.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}