{"id": "8234746878d4297cb07a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "missing_schema", "question": "Since no schema is provided by me, find tables before writing sql for inspections.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "8234746878d4297cb07a_en", "language": "en", "backend": "supabase", "operation": "missing_schema", "question": "Given that I have not provided a database schema, find the available tables before writing SQL for inspections.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "8234746878d4297cb07a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "missing_schema", "question": "Kripya inspections ke liye SQL likhne se pehle available tables ka pata lagao, kyunki maine database schema nahi diya hai.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "797213190eddac109b96_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sql_error", "question": "The last one failed because of a unknown column. Please inspect inspections schema before retrying.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "41f5fbcefb9d550068a0_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "missing_schema", "question": "Since no schema is provided by me, find tables before writing sql for specimens.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "4b73bea165fb5f9555f3_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sql_error", "question": "The last one failed because of a unknown column. Please inspect inspections schema before retrying.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "0a2ec84ba0e594d94b9e_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sql_error", "question": "The last one failed because of a unknown column. Please inspect reservations schema before retrying.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "44135783de9bd091bfe5_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "missing_schema", "question": "Since no schema is provided by me, find tables before writing sql for inspections.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_inspections", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_inspections\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6db4ac493194a52cbd3a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "missing_schema", "question": "Since no schema is provided by me, find tables before writing sql for reservations.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_reservations", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_reservations\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "797213190eddac109b96_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sql_error", "question": "Pichla query fail raha kyunki ek column unknown tha. Dobara try karne se pehle inspections schema verify karo.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "4b73bea165fb5f9555f3_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sql_error", "question": "Pichla query fail raha kyunki ek column unknown tha. Dobara try karne se pehle inspections schema verify karo.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "0a2ec84ba0e594d94b9e_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sql_error", "question": "Pichla query fail raha kyunki ek column unknown tha. Dobara try karne se pehle reservations schema verify karo.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "797213190eddac109b96_en", "language": "en", "backend": "supabase", "operation": "sql_error", "question": "An unknown column caused the last query to fail; check inspections schema before your next attempt.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "4b73bea165fb5f9555f3_en", "language": "en", "backend": "supabase", "operation": "sql_error", "question": "An unknown column caused the last query to fail; check inspections schema before your next attempt.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "0a2ec84ba0e594d94b9e_en", "language": "en", "backend": "supabase", "operation": "sql_error", "question": "An unknown column caused the last query to fail; check reservations schema before your next attempt.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "41f5fbcefb9d550068a0_en", "language": "en", "backend": "supabase", "operation": "missing_schema", "question": "Given that I have not provided a database schema, find the available tables before writing SQL for specimens.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "41f5fbcefb9d550068a0_hinglish", "language": "hinglish", "backend": "supabase", "operation": "missing_schema", "question": "Kripya specimens ke liye SQL likhne se pehle available tables ka pata lagao, kyunki maine database schema nahi diya hai.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "44135783de9bd091bfe5_en", "language": "en", "backend": "supabase", "operation": "missing_schema", "question": "Given that I have not provided a database schema, find the available tables before writing SQL for inspections.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_inspections", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_inspections\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6db4ac493194a52cbd3a_en", "language": "en", "backend": "supabase", "operation": "missing_schema", "question": "Given that I have not provided a database schema, find the available tables before writing SQL for reservations.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_reservations", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_reservations\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "44135783de9bd091bfe5_hinglish", "language": "hinglish", "backend": "supabase", "operation": "missing_schema", "question": "Kripya inspections ke liye SQL likhne se pehle available tables ka pata lagao, kyunki maine database schema nahi diya hai.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_inspections", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_inspections\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6db4ac493194a52cbd3a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "missing_schema", "question": "Kripya reservations ke liye SQL likhne se pehle available tables ka pata lagao, kyunki maine database schema nahi diya hai.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_reservations", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_reservations\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "84b16996198346dc9882_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "missing_schema", "question": "Since no schema is provided by me, find tables before writing sql for exhibits.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "5cb3a450bcdc957b7ba5_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "missing_schema", "question": "Since no schema is provided by me, find tables before writing sql for specimens.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "578a0a8a98b724c87050_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "list_tables", "question": "tell me the tables in this database", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "5689f342b9bf3175e2fe_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "all", "question": "fetch all rows from specimens", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "578a0a8a98b724c87050_en", "language": "en", "backend": "supabase", "operation": "list_tables", "question": "Display the schema tables in this database.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "578a0a8a98b724c87050_hinglish", "language": "hinglish", "backend": "supabase", "operation": "list_tables", "question": "Is database ki tables ka detail do.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "32876ff450ca027d658e_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "count", "question": "how many lines in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5689f342b9bf3175e2fe_en", "language": "en", "backend": "supabase", "operation": "all", "question": "Get all rows from specimens.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2ad3e251c936135e2005_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "all", "question": "fetch all rows from specimens", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "32876ff450ca027d658e_hinglish", "language": "hinglish", "backend": "supabase", "operation": "count", "question": "Batao specimens mein kitni entries hain.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "84b16996198346dc9882_en", "language": "en", "backend": "mysql", "operation": "missing_schema", "question": "Given that I have not provided a database schema, find the available tables before writing SQL for exhibits.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "2ad3e251c936135e2005_en", "language": "en", "backend": "supabase", "operation": "all", "question": "Get all rows from specimens.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "84b16996198346dc9882_hinglish", "language": "hinglish", "backend": "mysql", "operation": "missing_schema", "question": "Kripya exhibits ke liye SQL likhne se pehle available tables ka pata lagao, kyunki maine database schema nahi diya hai.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "5cb3a450bcdc957b7ba5_en", "language": "en", "backend": "mysql", "operation": "missing_schema", "question": "Given that I have not provided a database schema, find the available tables before writing SQL for specimens.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "772353a7da7eebcf3244_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "avg", "question": "determine the mean quantity in specimens", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(quantity) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(quantity) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e4a81e1122226c7b26d9_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "describe", "question": "Look up the columns and types for specimens.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "5689f342b9bf3175e2fe_hinglish", "language": "hinglish", "backend": "supabase", "operation": "all", "question": "specimens ke saare records get karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "efee65268e6dc2bb069e_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project", "question": "bas specimens ka customer_name do please", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT customer_name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0b8db53d711baed4f4ed_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gte", "question": "output records from specimens where cost is not below 10", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE cost >= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5cb3a450bcdc957b7ba5_hinglish", "language": "hinglish", "backend": "mysql", "operation": "missing_schema", "question": "Kripya specimens ke liye SQL likhne se pehle available tables ka pata lagao, kyunki maine database schema nahi diya hai.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "0ffff93d2f774a97e3f3_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "min", "question": "What is the floor price in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(price) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(price) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "efee65268e6dc2bb069e_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project", "question": "display customer_name from specimens and nothing else", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT customer_name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2e2ac3c5dcf95429daaa_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "null", "question": "Show me the rows from specimens where title is null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2e2ac3c5dcf95429daaa_hinglish", "language": "hinglish", "backend": "supabase", "operation": "null", "question": "specimens se title null wali rows select karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2ad3e251c936135e2005_hinglish", "language": "hinglish", "backend": "supabase", "operation": "all", "question": "specimens ke saare records get karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2984a6707d4e07f81695_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "write", "question": "Remove all records from exhibits now.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "772353a7da7eebcf3244_en", "language": "en", "backend": "supabase", "operation": "avg", "question": "Return the average quantity in specimens.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(quantity) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(quantity) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ad0842eb95739a0c88ec_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "avg", "question": "determine the mean balance in inspections", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e4a81e1122226c7b26d9_en", "language": "en", "backend": "supabase", "operation": "describe", "question": "Reveal the column structure and types for specimens.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "7a843b154fbd48aac50b_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "all", "question": "fetch all rows from inspections", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4f5c410b4b3379c58420_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "or", "question": "Display rows from specimens where category is pending or North.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE category IN ('pending', 'North');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE category IN ('pending', 'North');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0b8db53d711baed4f4ed_en", "language": "en", "backend": "supabase", "operation": "gte", "question": "Show the data from specimens where cost is 10 or above.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE cost >= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "80ed46f6e275ed01c6c1_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "count_eq", "question": "how many exhibits rows where category is Pune", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM exhibits WHERE category = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits WHERE category = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fea44ee93f1cea0883dd_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "count_eq", "question": "how many exhibits rows where category is Delhi", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM exhibits WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e3d12f1bf51bbf54a331_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "contains", "question": "List specimens rows where description equals North, ignore case differences.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE LOWER(description) LIKE LOWER('%North%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(description) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "96a74866d51ea7899cd8_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gte", "question": "output records from specimens where score is not below 20", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE score >= 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE score >= 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "772353a7da7eebcf3244_hinglish", "language": "hinglish", "backend": "supabase", "operation": "avg", "question": "specimens me quantity ka average return karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(quantity) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(quantity) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1395b37e5ba28fa3c134_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "min", "question": "What is the floor amount in inspections?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(amount) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(amount) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e4a81e1122226c7b26d9_hinglish", "language": "hinglish", "backend": "supabase", "operation": "describe", "question": "Mujhe specimens ke columns aur types ke baare mein info do.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "7a843b154fbd48aac50b_en", "language": "en", "backend": "supabase", "operation": "all", "question": "Get all rows from inspections.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2e2ac3c5dcf95429daaa_en", "language": "en", "backend": "supabase", "operation": "null", "question": "I want to view the rows in specimens where title is null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d735c1ac1979ff1e8620_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sum", "question": "i want the total price of inspections", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT SUM(price) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "28a7ce61b1233483bf73_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sort_desc", "question": "retrieve all records from specimens sorted by salary high to low", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c15c59356c7863f0b7e9_en", "language": "en", "backend": "supabase", "operation": "eq", "question": "Bring up all rows from specimens where city has the value Books.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE city = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE city = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c15c59356c7863f0b7e9_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "eq", "question": "Get the full list of rows in specimens where city is Books.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE city = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE city = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "358364da33322bbf17e2_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project_eq", "question": "give me title from exhibits where city is Pune", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT title FROM exhibits WHERE city = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT title FROM exhibits WHERE city = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ad0842eb95739a0c88ec_en", "language": "en", "backend": "supabase", "operation": "avg", "question": "Return the average balance in inspections.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ad0842eb95739a0c88ec_hinglish", "language": "hinglish", "backend": "supabase", "operation": "avg", "question": "inspections me balance ka average return karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "65666802d11552ba1fc2_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "write", "question": "Remove all records from inspections now.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "32876ff450ca027d658e_en", "language": "en", "backend": "supabase", "operation": "count", "question": "What is the volume of data rows in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0ffff93d2f774a97e3f3_en", "language": "en", "backend": "supabase", "operation": "min", "question": "What is the baseline price value in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(price) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(price) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "0ffff93d2f774a97e3f3_hinglish", "language": "hinglish", "backend": "supabase", "operation": "min", "question": "specimens mein price ka minimum reading kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(price) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(price) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4f5c410b4b3379c58420_en", "language": "en", "backend": "supabase", "operation": "or", "question": "Bring up rows from specimens where category is either pending or North.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE category IN ('pending', 'North');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE category IN ('pending', 'North');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4f5c410b4b3379c58420_hinglish", "language": "hinglish", "backend": "supabase", "operation": "or", "question": "specimens mein category pending ya North wali rows dikhao na.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE category IN ('pending', 'North');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE category IN ('pending', 'North');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "efee65268e6dc2bb069e_en", "language": "en", "backend": "supabase", "operation": "project", "question": "Display customer_name from specimens as the sole result.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT customer_name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0b8db53d711baed4f4ed_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gte", "question": "specimens se rows dikhao jahan cost 10 ya usse bada hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE cost >= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "84b620e6d482597eea06_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project", "question": "bas exhibits ka display_name do please", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT display_name FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2984a6707d4e07f81695_en", "language": "en", "backend": "supabase", "operation": "write", "question": "Expunge every row from exhibits.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "2984a6707d4e07f81695_hinglish", "language": "hinglish", "backend": "supabase", "operation": "write", "question": "exhibits ka pura content hatwa do.", "expected": {"action": "clarify", "question": "Yeh connection read-only hai. Kya SELECT query chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"Yeh connection read-only hai. Kya SELECT query chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "96a74866d51ea7899cd8_en", "language": "en", "backend": "supabase", "operation": "gte", "question": "Show the data from specimens where score is 20 or above.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE score >= 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE score >= 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8234746878d4297cb07a_hi", "language": "hi", "backend": "supabase", "operation": "missing_schema", "question": "कृपया inspections के लिए SQL लिखने से पहले उपलब्ध तालिकाओं का पता लगाएं, क्योंकि मैंने डेटाबेस स्कीमा नहीं दी है।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "358364da33322bbf17e2_en", "language": "en", "backend": "supabase", "operation": "project_eq", "question": "List the title from exhibits where city is Pune.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT title FROM exhibits WHERE city = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT title FROM exhibits WHERE city = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1395b37e5ba28fa3c134_hinglish", "language": "hinglish", "backend": "supabase", "operation": "min", "question": "inspections mein amount ka minimum reading kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(amount) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(amount) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7a843b154fbd48aac50b_hinglish", "language": "hinglish", "backend": "supabase", "operation": "all", "question": "inspections ke saare records get karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "80ed46f6e275ed01c6c1_en", "language": "en", "backend": "supabase", "operation": "count_eq", "question": "Provide the row count for exhibits where category is Pune.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM exhibits WHERE category = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits WHERE category = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "80ed46f6e275ed01c6c1_hinglish", "language": "hinglish", "backend": "supabase", "operation": "count_eq", "question": "Please count the rows in exhibits where category is Pune.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM exhibits WHERE category = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits WHERE category = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d735c1ac1979ff1e8620_en", "language": "en", "backend": "supabase", "operation": "sum", "question": "Output the total price for the inspections entity.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT SUM(price) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "28a7ce61b1233483bf73_en", "language": "en", "backend": "supabase", "operation": "sort_desc", "question": "Display the entire table specimens sorted by salary from highest to lowest.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "84b620e6d482597eea06_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project", "question": "display display_name from exhibits and nothing else", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT display_name FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bcd7a532a06274ddddee_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "or", "question": "Display rows from exhibits where status is pending or South.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE status IN ('pending', 'South');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('pending', 'South');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fea44ee93f1cea0883dd_en", "language": "en", "backend": "supabase", "operation": "count_eq", "question": "Provide the row count for exhibits where category is Delhi.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM exhibits WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fea44ee93f1cea0883dd_hinglish", "language": "hinglish", "backend": "supabase", "operation": "count_eq", "question": "Please count the rows in exhibits where category is Delhi.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM exhibits WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8a456280df4a711c3cda_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "year", "question": "Fetch inspections entries where created_at year is 2024.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE EXTRACT(YEAR FROM created_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE EXTRACT(YEAR FROM created_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c15c59356c7863f0b7e9_hinglish", "language": "hinglish", "backend": "supabase", "operation": "eq", "question": "specimens se wo saari rows lao jahan city ka value Books hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE city = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE city = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a7afc1ab21c25c700603_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "or", "question": "Display rows from exhibits where status is Mumbai or Electronics.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE status IN ('Mumbai', 'Electronics');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('Mumbai', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1395b37e5ba28fa3c134_en", "language": "en", "backend": "supabase", "operation": "min", "question": "What is the baseline amount value in inspections?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(amount) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(amount) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "71349134f8ae16809501_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sort_desc", "question": "retrieve all records from exhibits sorted by salary high to low", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ac83eae59947f137269f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "write", "question": "Remove all records from reservations now.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "28a7ce61b1233483bf73_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sort_desc", "question": "specimens ki poori table salary highest se lowest order mein dikhao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1a1af4deb475f0224a59_en", "language": "en", "backend": "supabase", "operation": "bottom", "question": "Present the 10 minimal price entries from inspections in ascending order.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections ORDER BY price ASC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY price ASC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1a1af4deb475f0224a59_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "bottom", "question": "print the 10 minimal price rows in inspections from lowest up", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections ORDER BY price ASC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY price ASC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bcd7a532a06274ddddee_hinglish", "language": "hinglish", "backend": "supabase", "operation": "or", "question": "exhibits mein status pending ya South wali rows dikhao na.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE status IN ('pending', 'South');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('pending', 'South');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7dd163238936d96d0bd4_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "distinct", "question": "i need the distinct city values from reservations", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT city FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8a456280df4a711c3cda_en", "language": "en", "backend": "supabase", "operation": "year", "question": "Show me the inspections rows for 2024 using created_at.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE EXTRACT(YEAR FROM created_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE EXTRACT(YEAR FROM created_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8a456280df4a711c3cda_hinglish", "language": "hinglish", "backend": "supabase", "operation": "year", "question": "inspections mein created_at mein 2024 wale records dikhao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE EXTRACT(YEAR FROM created_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE EXTRACT(YEAR FROM created_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "96a74866d51ea7899cd8_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gte", "question": "specimens se rows dikhao jahan score 20 ya usse bada hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE score >= 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE score >= 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4745f53221e07d4f5ae1_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "contains", "question": "List specimens rows where display_name equals Pune, ignore case differences.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE LOWER(display_name) LIKE LOWER('%Pune%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(display_name) LIKE LOWER('%Pune%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "358364da33322bbf17e2_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project_eq", "question": "exhibits se title ki list do jahan city Pune hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT title FROM exhibits WHERE city = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT title FROM exhibits WHERE city = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a7afc1ab21c25c700603_hinglish", "language": "hinglish", "backend": "supabase", "operation": "or", "question": "exhibits mein status Mumbai ya Electronics wali rows dikhao na.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE status IN ('Mumbai', 'Electronics');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('Mumbai', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6b9bacf4d9aa380af183_en", "language": "en", "backend": "supabase", "operation": "ambiguous", "question": "Print the best available records in reservations.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "6b9bacf4d9aa380af183_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ambiguous", "question": "Give me the top records in reservations.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "6b9bacf4d9aa380af183_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ambiguous", "question": "reservations mein available best records print karo.", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "65666802d11552ba1fc2_en", "language": "en", "backend": "supabase", "operation": "write", "question": "Expunge every row from inspections.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "65666802d11552ba1fc2_hinglish", "language": "hinglish", "backend": "supabase", "operation": "write", "question": "inspections ka pura content hatwa do.", "expected": {"action": "clarify", "question": "Yeh connection read-only hai. Kya SELECT query chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"Yeh connection read-only hai. Kya SELECT query chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "e91a7141af66518c19a2_en", "language": "en", "backend": "supabase", "operation": "max", "question": "What is the largest number for cost in inspections?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(cost) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(cost) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e91a7141af66518c19a2_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "max", "question": "what is the upper bound of cost in inspections?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(cost) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(cost) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dd6cdefd15edcdcddb76_en", "language": "en", "backend": "supabase", "operation": "bottom", "question": "Present the 5 minimal salary entries from exhibits in ascending order.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY salary ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dd6cdefd15edcdcddb76_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "bottom", "question": "print the 5 minimal salary rows in exhibits from lowest up", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY salary ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d735c1ac1979ff1e8620_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sum", "question": "inspections entity ke liye price ka total output do.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT SUM(price) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c8fd1555fd64b08f1241_en", "language": "en", "backend": "supabase", "operation": "eq", "question": "Bring up all rows from inspections where city has the value Electronics.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE city = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE city = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c8fd1555fd64b08f1241_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "eq", "question": "Get the full list of rows in inspections where city is Electronics.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE city = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE city = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "eb982cf071094adaca4f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project_eq", "question": "give me name from reservations where category is Delhi", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT name FROM reservations WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bcd7a532a06274ddddee_en", "language": "en", "backend": "supabase", "operation": "or", "question": "Bring up rows from exhibits where status is either pending or South.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE status IN ('pending', 'South');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('pending', 'South');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e3d12f1bf51bbf54a331_hinglish", "language": "hinglish", "backend": "supabase", "operation": "contains", "question": "Output karo specimens mein description mein North match hone wali rows, case insensitive raho.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE LOWER(description) LIKE LOWER('%North%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(description) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "772353a7da7eebcf3244_hi", "language": "hi", "backend": "supabase", "operation": "avg", "question": "specimens में quantity का औसत लौटाएँ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(quantity) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(quantity) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a7afc1ab21c25c700603_en", "language": "en", "backend": "supabase", "operation": "or", "question": "Bring up rows from exhibits where status is either Mumbai or Electronics.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE status IN ('Mumbai', 'Electronics');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('Mumbai', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "71349134f8ae16809501_en", "language": "en", "backend": "supabase", "operation": "sort_desc", "question": "Display the entire table exhibits sorted by salary from highest to lowest.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "71349134f8ae16809501_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sort_desc", "question": "exhibits ki poori table salary highest se lowest order mein dikhao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e91a7141af66518c19a2_hinglish", "language": "hinglish", "backend": "supabase", "operation": "max", "question": "inspections mein cost ka sabse bada number kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(cost) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(cost) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ac83eae59947f137269f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "write", "question": "reservations ka pura content hatwa do.", "expected": {"action": "clarify", "question": "Yeh connection read-only hai. Kya SELECT query chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"Yeh connection read-only hai. Kya SELECT query chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "c8fd1555fd64b08f1241_hinglish", "language": "hinglish", "backend": "supabase", "operation": "eq", "question": "inspections se wo saari rows lao jahan city ka value Electronics hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE city = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE city = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "eb982cf071094adaca4f_en", "language": "en", "backend": "supabase", "operation": "project_eq", "question": "List the name from reservations where category is Delhi.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT name FROM reservations WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "84b620e6d482597eea06_en", "language": "en", "backend": "supabase", "operation": "project", "question": "Display display_name from exhibits as the sole result.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT display_name FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "92e36c8a6ca6e7aa2298_en", "language": "en", "backend": "supabase", "operation": "group_count", "question": "Find the count of rows per category in reservations.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "92e36c8a6ca6e7aa2298_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "group_count", "question": "give me the count for each category rows in reservations", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7d18561ec942f18f1299_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "count_eq", "question": "how many reservations rows where status is O'Reilly", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM reservations WHERE status = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM reservations WHERE status = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ad0842eb95739a0c88ec_hi", "language": "hi", "backend": "supabase", "operation": "avg", "question": "inspections में balance का औसत लौटाएँ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4dac1dcf2ef14bfee2d2_en", "language": "en", "backend": "supabase", "operation": "group_count", "question": "Find the count of rows per category in reservations.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4dac1dcf2ef14bfee2d2_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "group_count", "question": "give me the count for each category rows in reservations", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ac83eae59947f137269f_en", "language": "en", "backend": "supabase", "operation": "write", "question": "Expunge every row from reservations.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "efee65268e6dc2bb069e_hi", "language": "hi", "backend": "supabase", "operation": "project", "question": "सिर्फ specimens का customer_name दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT customer_name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "eb982cf071094adaca4f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project_eq", "question": "reservations se name ki list do jahan category Delhi hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT name FROM reservations WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b30330bb2c17b32c6883_en", "language": "en", "backend": "supabase", "operation": "sort_asc", "question": "View all records in exhibits sorted by cost from least to greatest.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY cost ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "060573c57829f3ce6e8f_en", "language": "en", "backend": "supabase", "operation": "not_null", "question": "Present the data from exhibits where item_name is not null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE item_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE item_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "060573c57829f3ce6e8f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "not_null", "question": "Find the rows in exhibits where item_name is not null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE item_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE item_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "92e36c8a6ca6e7aa2298_hinglish", "language": "hinglish", "backend": "supabase", "operation": "group_count", "question": "reservations me pratyek category ke liye rows ki sankhya dhundo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e3d12f1bf51bbf54a331_en", "language": "en", "backend": "supabase", "operation": "contains", "question": "List all rows in specimens where description is equal to North in a case-insensitive way.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE LOWER(description) LIKE LOWER('%North%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(description) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fd2a795a1557dc1487c6_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "or", "question": "Display rows from reservations where city is Electronics or Pune.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE city IN ('Electronics', 'Pune');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE city IN ('Electronics', 'Pune');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fd2a795a1557dc1487c6_hinglish", "language": "hinglish", "backend": "supabase", "operation": "or", "question": "reservations mein city Electronics ya Pune wali rows dikhao na.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE city IN ('Electronics', 'Pune');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE city IN ('Electronics', 'Pune');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3ad8170567694db8c8f9_en", "language": "en", "backend": "supabase", "operation": "not_null", "question": "Present the data from inspections where display_name is not null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE display_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE display_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3ad8170567694db8c8f9_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "not_null", "question": "Find the rows in inspections where display_name is not null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE display_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE display_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5b5b87a5a58feab9d244_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "year", "question": "Fetch reservations entries where recorded_at year is 2021.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE EXTRACT(YEAR FROM recorded_at) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5b5b87a5a58feab9d244_hinglish", "language": "hinglish", "backend": "supabase", "operation": "year", "question": "reservations mein recorded_at mein 2021 wale records dikhao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE EXTRACT(YEAR FROM recorded_at) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "37a16740c32af37d3b15_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lte", "question": "show the reservations table rows where score is at most 50", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE score <= 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score <= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0c15b441e3e3ee8bb0df_en", "language": "en", "backend": "supabase", "operation": "not_null", "question": "Present the data from reservations where name is not null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0c15b441e3e3ee8bb0df_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "not_null", "question": "Find the rows in reservations where name is not null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0c15b441e3e3ee8bb0df_hinglish", "language": "hinglish", "backend": "supabase", "operation": "not_null", "question": "reservations ka data present karo jahan name null nahi hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4dac1dcf2ef14bfee2d2_hinglish", "language": "hinglish", "backend": "supabase", "operation": "group_count", "question": "reservations me pratyek category ke liye rows ki sankhya dhundo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "56189c5cdc014dd771be_en", "language": "en", "backend": "supabase", "operation": "lt", "question": "Query reservations for rows where balance is less than 50.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE balance < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ec5a4c33f3a4c1fdfef3_en", "language": "en", "backend": "supabase", "operation": "group_sum", "question": "Please provide the total rating for each city in reservations.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT city, SUM(rating) FROM reservations GROUP BY city;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT city, SUM(rating) FROM reservations GROUP BY city;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "26db633822b8319f83ec_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "contains", "question": "List reservations rows where name equals active, ignore case differences.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE LOWER(name) LIKE LOWER('%active%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE LOWER(name) LIKE LOWER('%active%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "060573c57829f3ce6e8f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "not_null", "question": "exhibits ka data present karo jahan item_name null nahi hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE item_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE item_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7dd163238936d96d0bd4_en", "language": "en", "backend": "supabase", "operation": "distinct", "question": "I want to see the distinct city values available in reservations.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT city FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7dd163238936d96d0bd4_hinglish", "language": "hinglish", "backend": "supabase", "operation": "distinct", "question": "Mujhe reservations mein available city ke distinct values dekhne hain.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT city FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3ad8170567694db8c8f9_hinglish", "language": "hinglish", "backend": "supabase", "operation": "not_null", "question": "inspections ka data present karo jahan display_name null nahi hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE display_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE display_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5b5b87a5a58feab9d244_en", "language": "en", "backend": "supabase", "operation": "year", "question": "Show me the reservations rows for 2021 using recorded_at.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE EXTRACT(YEAR FROM recorded_at) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "37a16740c32af37d3b15_en", "language": "en", "backend": "supabase", "operation": "lte", "question": "Query reservations for rows where score is not greater than 50.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE score <= 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score <= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "32876ff450ca027d658e_hi", "language": "hi", "backend": "supabase", "operation": "count", "question": "specimens में कितनी लाइनें मौजूद हैं?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "175564da35a6c139e5d8_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "contains", "question": "List inspections rows where display_name equals North, ignore case differences.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE LOWER(display_name) LIKE LOWER('%North%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE LOWER(display_name) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ec5a4c33f3a4c1fdfef3_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "group_sum", "question": "I need to see the total rating for each city in reservations", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT city, SUM(rating) FROM reservations GROUP BY city;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT city, SUM(rating) FROM reservations GROUP BY city;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ec5a4c33f3a4c1fdfef3_hinglish", "language": "hinglish", "backend": "supabase", "operation": "group_sum", "question": "reservations me city ke hisaab se rating ka total do please.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT city, SUM(rating) FROM reservations GROUP BY city;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT city, SUM(rating) FROM reservations GROUP BY city;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c4f25575abfd24e3cd63_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lte", "question": "show the reservations table rows where cost is at most 1", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE cost <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b30330bb2c17b32c6883_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sort_asc", "question": "can i see all rows from exhibits sorted by cost from small to large", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY cost ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b30330bb2c17b32c6883_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sort_asc", "question": "exhibits se saara data cost ke badhte hue order mein lo aur dikhao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY cost ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "645fd36c186709e64922_en", "language": "en", "backend": "supabase", "operation": "between", "question": "Select all items from specimens where the amount column has a value between 1000 and 1010, inclusive.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE amount BETWEEN 1000 AND 1010;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE amount BETWEEN 1000 AND 1010;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fd2a795a1557dc1487c6_en", "language": "en", "backend": "supabase", "operation": "or", "question": "Bring up rows from reservations where city is either Electronics or Pune.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE city IN ('Electronics', 'Pune');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE city IN ('Electronics', 'Pune');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4745f53221e07d4f5ae1_hinglish", "language": "hinglish", "backend": "supabase", "operation": "contains", "question": "Output karo specimens mein display_name mein Pune match hone wali rows, case insensitive raho.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE LOWER(display_name) LIKE LOWER('%Pune%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(display_name) LIKE LOWER('%Pune%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "37a16740c32af37d3b15_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lte", "question": "reservations se wo rows dhundo jaha score 50 se bada nahi hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE score <= 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score <= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7d18561ec942f18f1299_en", "language": "en", "backend": "supabase", "operation": "count_eq", "question": "Provide the row count for reservations where status is O'Reilly.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM reservations WHERE status = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM reservations WHERE status = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7d18561ec942f18f1299_hinglish", "language": "hinglish", "backend": "supabase", "operation": "count_eq", "question": "Please count the rows in reservations where status is O'Reilly.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM reservations WHERE status = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM reservations WHERE status = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0ce22f4c8bc7554d586d_en", "language": "en", "backend": "supabase", "operation": "lt", "question": "Query reservations for rows where score is less than 50.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE score < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f35f7a78a14e903920ee_en", "language": "en", "backend": "supabase", "operation": "month", "question": "Select entries from reservations where order_date corresponds to 3.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE EXTRACT(MONTH FROM order_date) = 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(MONTH FROM order_date) = 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c4f25575abfd24e3cd63_en", "language": "en", "backend": "supabase", "operation": "lte", "question": "Query reservations for rows where cost is not greater than 1.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE cost <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "29bdf95ec36d6924a576_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "contains", "question": "List exhibits rows where item_name equals O'Reilly, ignore case differences.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE LOWER(item_name) LIKE LOWER('%O''Reilly%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE LOWER(item_name) LIKE LOWER('%O''Reilly%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "56189c5cdc014dd771be_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lt", "question": "can you select the rows from reservations where balance is less than 50?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE balance < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "56189c5cdc014dd771be_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lt", "question": "reservations se un rows ko query karo jahan balance 50 se kam hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE balance < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0b8689cf8e381cbcfe86_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "and", "question": "find rows in reservations where department is pending and price is above 10000", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE department = 'pending' AND price > 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE department = 'pending' AND price > 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2e2ac3c5dcf95429daaa_hi", "language": "hi", "backend": "supabase", "operation": "null", "question": "specimens से title null वाली पंक्तियाँ चुनें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1a1af4deb475f0224a59_hinglish", "language": "hinglish", "backend": "supabase", "operation": "bottom", "question": "price ke badhte order me inspections ki 10 sabse kam rows, lowest se shuru karke, lao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections ORDER BY price ASC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY price ASC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f35f7a78a14e903920ee_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "month", "question": "extract rows from reservations where order_date month is 3", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE EXTRACT(MONTH FROM order_date) = 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(MONTH FROM order_date) = 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c4f25575abfd24e3cd63_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lte", "question": "reservations se wo rows dhundo jaha cost 1 se bada nahi hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE cost <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "84b620e6d482597eea06_hi", "language": "hi", "backend": "supabase", "operation": "project", "question": "सिर्फ exhibits का display_name दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT display_name FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "645fd36c186709e64922_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "between", "question": "pull the rows from specimens where amount is greater than or equal to 1000 and less than or equal to 1010", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE amount BETWEEN 1000 AND 1010;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE amount BETWEEN 1000 AND 1010;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4745f53221e07d4f5ae1_en", "language": "en", "backend": "supabase", "operation": "contains", "question": "List all rows in specimens where display_name is equal to Pune in a case-insensitive way.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE LOWER(display_name) LIKE LOWER('%Pune%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(display_name) LIKE LOWER('%Pune%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "358364da33322bbf17e2_hi", "language": "hi", "backend": "supabase", "operation": "project_eq", "question": "exhibits से title की सूची दें जहाँ city Pune है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT title FROM exhibits WHERE city = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT title FROM exhibits WHERE city = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dd6cdefd15edcdcddb76_hinglish", "language": "hinglish", "backend": "supabase", "operation": "bottom", "question": "salary ke badhte order me exhibits ki 5 sabse kam rows, lowest se shuru karke, lao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY salary ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5689f342b9bf3175e2fe_hi", "language": "hi", "backend": "supabase", "operation": "all", "question": "specimens के सभी रिकॉर्ड प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0abe13c841fd7c3ac885_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "having", "question": "Tell me the region groups in inspections with more than 5000 rows and their count values.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT region, COUNT(*) FROM inspections GROUP BY region HAVING COUNT(*) > 5000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM inspections GROUP BY region HAVING COUNT(*) > 5000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0abe13c841fd7c3ac885_hinglish", "language": "hinglish", "backend": "supabase", "operation": "having", "question": "Please inspections me 5000 se zyada rows wale region groups aur unki count ko display karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT region, COUNT(*) FROM inspections GROUP BY region HAVING COUNT(*) > 5000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM inspections GROUP BY region HAVING COUNT(*) > 5000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0b8689cf8e381cbcfe86_en", "language": "en", "backend": "supabase", "operation": "and", "question": "Show me the reservations rows where department is pending and price exceeds 10000.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE department = 'pending' AND price > 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE department = 'pending' AND price > 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "916ff7ce3eea9aa64393_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "missing_value", "question": "execute filter on reservations for kind, but i have not specified a value", "expected": {"action": "clarify", "question": "Which value should kind equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should kind equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "2984a6707d4e07f81695_hi", "language": "hi", "backend": "supabase", "operation": "write", "question": "exhibits का पूरा कंटेंट डिलीट करें।", "expected": {"action": "clarify", "question": "यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "0ce22f4c8bc7554d586d_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lt", "question": "can you select the rows from reservations where score is less than 50?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE score < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0ce22f4c8bc7554d586d_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lt", "question": "reservations se un rows ko query karo jahan score 50 se kam hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE score < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f35f7a78a14e903920ee_hinglish", "language": "hinglish", "backend": "supabase", "operation": "month", "question": "Show me the data from reservations where order_date ka month 3 hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE EXTRACT(MONTH FROM order_date) = 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(MONTH FROM order_date) = 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "26db633822b8319f83ec_hinglish", "language": "hinglish", "backend": "supabase", "operation": "contains", "question": "Output karo reservations mein name mein active match hone wali rows, case insensitive raho.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE LOWER(name) LIKE LOWER('%active%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE LOWER(name) LIKE LOWER('%active%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2ad3e251c936135e2005_hi", "language": "hi", "backend": "supabase", "operation": "all", "question": "specimens के सभी रिकॉर्ड प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "645fd36c186709e64922_hinglish", "language": "hinglish", "backend": "supabase", "operation": "between", "question": "specimens se wo saare items select karo jahan amount column ka value 1000 aur 1010 ke beech hai, inclusive.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE amount BETWEEN 1000 AND 1010;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE amount BETWEEN 1000 AND 1010;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e4a81e1122226c7b26d9_hi", "language": "hi", "backend": "supabase", "operation": "describe", "question": "टेबल specimens के कॉलम संरचना और टाइप्स दिखाइए।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "e9cfe7f6c91a2434d315_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "final", "question": "db tool returned 102 rows. tell me the total.", "expected": {"action": "answer", "text": "The tool returned 102 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 102 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "e9cfe7f6c91a2434d315_hinglish", "language": "hinglish", "backend": "supabase", "operation": "final", "question": "Kitni rows aayin database tool se? Count hai 102.", "expected": {"action": "answer", "text": "Tool ne 102 rows return ki."}, "output": "{\"action\":\"answer\",\"text\":\"Tool ne 102 rows return ki.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "175564da35a6c139e5d8_hinglish", "language": "hinglish", "backend": "supabase", "operation": "contains", "question": "Output karo inspections mein display_name mein North match hone wali rows, case insensitive raho.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE LOWER(display_name) LIKE LOWER('%North%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE LOWER(display_name) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0b8689cf8e381cbcfe86_hinglish", "language": "hinglish", "backend": "supabase", "operation": "and", "question": "reservations mein wo rows dhundo jahan department pending hai aur price 10000 se upar hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE department = 'pending' AND price > 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE department = 'pending' AND price > 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d735c1ac1979ff1e8620_hi", "language": "hi", "backend": "supabase", "operation": "sum", "question": "inspections में price के सभी मानों का योग करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT SUM(price) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "eb982cf071094adaca4f_hi", "language": "hi", "backend": "supabase", "operation": "project_eq", "question": "reservations से name की सूची दें जहाँ category Delhi है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT name FROM reservations WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "85e339da2fb28bae5395_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "date_after", "question": "Query rows in reservations where order_date is 2024-10-15 onwards.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE order_date >= '2024-10-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE order_date >= '2024-10-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "65666802d11552ba1fc2_hi", "language": "hi", "backend": "supabase", "operation": "write", "question": "inspections का पूरा कंटेंट डिलीट करें।", "expected": {"action": "clarify", "question": "यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "e91a7141af66518c19a2_hi", "language": "hi", "backend": "supabase", "operation": "max", "question": "inspections में cost का सबसे बड़ा नंबर क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(cost) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(cost) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0ffff93d2f774a97e3f3_hi", "language": "hi", "backend": "supabase", "operation": "min", "question": "specimens में price का न्यूनतम परिमाण क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(price) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(price) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "26db633822b8319f83ec_en", "language": "en", "backend": "supabase", "operation": "contains", "question": "List all rows in reservations where name is equal to active in a case-insensitive way.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE LOWER(name) LIKE LOWER('%active%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE LOWER(name) LIKE LOWER('%active%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "30dd94e63a065d87109e_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sql_error", "question": "The last one failed because of a unknown column. Please inspect specimens schema before retrying.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "specimens"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"specmens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "578a0a8a98b724c87050_hi", "language": "hi", "backend": "supabase", "operation": "list_tables", "question": "इस डेटाबेस की तालिकाओं का विवरण दीजिए।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "916ff7ce3eea9aa64393_en", "language": "en", "backend": "supabase", "operation": "missing_value", "question": "Execute a filter on reservations based on kind, but I have not specified a value.", "expected": {"action": "clarify", "question": "Which value should kind equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should kind equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "1395b37e5ba28fa3c134_hi", "language": "hi", "backend": "supabase", "operation": "min", "question": "inspections में amount का न्यूनतम परिमाण क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(amount) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(amount) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "29bdf95ec36d6924a576_hinglish", "language": "hinglish", "backend": "supabase", "operation": "contains", "question": "Output karo exhibits mein item_name mein O'Reilly match hone wali rows, case insensitive raho.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE LOWER(item_name) LIKE LOWER('%O''Reilly%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE LOWER(item_name) LIKE LOWER('%O''Reilly%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6f162db917ded36d0df0_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sql_error", "question": "The last one failed because of a unknown column. Please inspect specimens schema before retrying.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "specimens"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"specmens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "175564da35a6c139e5d8_en", "language": "en", "backend": "supabase", "operation": "contains", "question": "List all rows in inspections where display_name is equal to North in a case-insensitive way.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE LOWER(display_name) LIKE LOWER('%North%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE LOWER(display_name) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ac83eae59947f137269f_hi", "language": "hi", "backend": "supabase", "operation": "write", "question": "reservations का पूरा कंटेंट डिलीट करें।", "expected": {"action": "clarify", "question": "यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "e9cfe7f6c91a2434d315_en", "language": "en", "backend": "supabase", "operation": "final", "question": "The database tool returned 102 rows. Can you tell me the number?", "expected": {"action": "answer", "text": "The tool returned 102 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 102 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "7a843b154fbd48aac50b_hi", "language": "hi", "backend": "supabase", "operation": "all", "question": "inspections के सभी रिकॉर्ड प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4f5c410b4b3379c58420_hi", "language": "hi", "backend": "supabase", "operation": "or", "question": "specimens में category pending या North वाली पंक्तियाँ दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE category IN ('pending', 'North');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE category IN ('pending', 'North');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0abe13c841fd7c3ac885_en", "language": "en", "backend": "supabase", "operation": "having", "question": "Give me the region groups in inspections where the row count is higher than 5000, along with the count values.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT region, COUNT(*) FROM inspections GROUP BY region HAVING COUNT(*) > 5000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM inspections GROUP BY region HAVING COUNT(*) > 5000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "940596942f7c5e4bf758_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "list_tables", "question": "tell me the tables in this database", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "0c15b441e3e3ee8bb0df_hi", "language": "hi", "backend": "supabase", "operation": "not_null", "question": "reservations का डेटा दिखाएं जहाँ name null नहीं है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "29bdf95ec36d6924a576_en", "language": "en", "backend": "supabase", "operation": "contains", "question": "List all rows in exhibits where item_name is equal to O'Reilly in a case-insensitive way.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE LOWER(item_name) LIKE LOWER('%O''Reilly%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE LOWER(item_name) LIKE LOWER('%O''Reilly%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "30dd94e63a065d87109e_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sql_error", "question": "Pichla query fail raha kyunki ek column unknown tha. Dobara try karne se pehle specimens schema verify karo.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "specimens"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"sicmens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "060573c57829f3ce6e8f_hi", "language": "hi", "backend": "supabase", "operation": "not_null", "question": "exhibits का डेटा दिखाएं जहाँ item_name null नहीं है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE item_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE item_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "85e339da2fb28bae5395_en", "language": "en", "backend": "supabase", "operation": "date_after", "question": "Extract rows from reservations where order_date falls on 2024-10-15 or beyond.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE order_date >= '2024-10-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE order_date >= '2024-10-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3ad8170567694db8c8f9_hi", "language": "hi", "backend": "supabase", "operation": "not_null", "question": "inspections का डेटा दिखाएं जहाँ display_name null नहीं है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE display_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE display_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "940596942f7c5e4bf758_en", "language": "en", "backend": "supabase", "operation": "list_tables", "question": "Display the schema tables in this database.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "940596942f7c5e4bf758_hinglish", "language": "hinglish", "backend": "supabase", "operation": "list_tables", "question": "Is database ki tables ka detail do.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "6f162db917ded36d0df0_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sql_error", "question": "Pichla query fail raha kyunki ek column unknown tha. Dobara try karne se pehle specimens schema verify karo.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "specimens"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"saarmens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "5a8f2e207d0efe5b018c_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "list_tables", "question": "tell me the tables in this database", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "30dd94e63a065d87109e_en", "language": "en", "backend": "mysql", "operation": "sql_error", "question": "An unknown column caused the last query to fail; check specimens schema before your next attempt.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "specimens"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"specmens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "916ff7ce3eea9aa64393_hinglish", "language": "hinglish", "backend": "supabase", "operation": "missing_value", "question": "kind ke basis pe reservations pe ek filter execute karo, lekin maine koi value specify nahi ki hai.", "expected": {"action": "clarify", "question": "kind ki kaunsi value chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"kind ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "bcd7a532a06274ddddee_hi", "language": "hi", "backend": "supabase", "operation": "or", "question": "exhibits में status pending या South वाली पंक्तियाँ दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE status IN ('pending', 'South');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('pending', 'South');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8a456280df4a711c3cda_hi", "language": "hi", "backend": "supabase", "operation": "year", "question": "inspections में created_at में 2024 वाले रिकॉर्ड दिखाएं।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE EXTRACT(YEAR FROM created_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE EXTRACT(YEAR FROM created_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "aa4f0e1f29573e237ee1_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "list_tables", "question": "tell me the tables in this database", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "a7afc1ab21c25c700603_hi", "language": "hi", "backend": "supabase", "operation": "or", "question": "exhibits में status Mumbai या Electronics वाली पंक्तियाँ दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE status IN ('Mumbai', 'Electronics');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('Mumbai', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6f162db917ded36d0df0_en", "language": "en", "backend": "mysql", "operation": "sql_error", "question": "An unknown column caused the last query to fail; check specimens schema before your next attempt.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "specimens"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"specmens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "ec5a4c33f3a4c1fdfef3_hi", "language": "hi", "backend": "supabase", "operation": "group_sum", "question": "reservations में city के अनुसार rating का कुल कृपया दें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT city, SUM(rating) FROM reservations GROUP BY city;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT city, SUM(rating) FROM reservations GROUP BY city;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5a8f2e207d0efe5b018c_en", "language": "en", "backend": "supabase", "operation": "list_tables", "question": "Display the schema tables in this database.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "5a8f2e207d0efe5b018c_hinglish", "language": "hinglish", "backend": "supabase", "operation": "list_tables", "question": "Is database ki tables ka detail do.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "aa4f0e1f29573e237ee1_en", "language": "en", "backend": "supabase", "operation": "list_tables", "question": "Display the schema tables in this database.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "aa4f0e1f29573e237ee1_hinglish", "language": "hinglish", "backend": "supabase", "operation": "list_tables", "question": "Is database ki tables ka detail do.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "c15c59356c7863f0b7e9_hi", "language": "hi", "backend": "supabase", "operation": "eq", "question": "specimens से वे सभी पंक्तियाँ लाएँ जहाँ city का मान Books है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE city = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE city = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8ebc9e868ee558a8ddfe_en", "language": "en", "backend": "supabase", "operation": "ambiguous", "question": "Print the best available records in specimens.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "8ebc9e868ee558a8ddfe_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ambiguous", "question": "Give me the top records in specimens.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "0b8db53d711baed4f4ed_hi", "language": "hi", "backend": "supabase", "operation": "gte", "question": "specimens से वे पंक्तियाँ दिखाएँ जहाँ cost 10 या उससे बड़ा है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE cost >= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b564e1a7b1e7fc6d2ab4_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "avg", "question": "determine the mean score in specimens", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT AVG(score) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT AVG(score) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "85e339da2fb28bae5395_hinglish", "language": "hinglish", "backend": "supabase", "operation": "date_after", "question": "reservations mein se wo rows chuno jinke order_date mein 2024-10-15 ya uske baad ki date hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE order_date >= '2024-10-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE order_date >= '2024-10-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "28530a7607feca135b73_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sum", "question": "i want the total price of specimens", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT SUM(price) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT SUM(price) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "fd2a795a1557dc1487c6_hi", "language": "hi", "backend": "supabase", "operation": "or", "question": "reservations में city Electronics या Pune वाली पंक्तियाँ दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE city IN ('Electronics', 'Pune');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE city IN ('Electronics', 'Pune');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5b5b87a5a58feab9d244_hi", "language": "hi", "backend": "supabase", "operation": "year", "question": "reservations में recorded_at में 2021 वाले रिकॉर्ड दिखाएं।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE EXTRACT(YEAR FROM recorded_at) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8ebc9e868ee558a8ddfe_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ambiguous", "question": "specimens mein available best records print karo.", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "7e1cd79fe94cad9c7a9e_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "count_eq", "question": "how many specimens rows where region is Books", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT COUNT(*) FROM specimens WHERE region = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT COUNT(*) FROM specimens WHERE region = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "ac6cc6f05e6d013971e5_en", "language": "en", "backend": "supabase", "operation": "max", "question": "What is the largest number for quantity in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT MAX(quantity) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT MAX(quantity) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "ac6cc6f05e6d013971e5_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "max", "question": "what is the upper bound of quantity in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT MAX(quantity) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT MAX(quantity) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "35406fa024e66bcb5dd7_en", "language": "en", "backend": "supabase", "operation": "lt", "question": "Query specimens for rows where score is less than 10000.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE score < 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT * FROM specimens WHERE score < 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "c8fd1555fd64b08f1241_hi", "language": "hi", "backend": "supabase", "operation": "eq", "question": "inspections से वे सभी पंक्तियाँ लाएँ जहाँ city का मान Electronics है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE city = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE city = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b564e1a7b1e7fc6d2ab4_en", "language": "en", "backend": "supabase", "operation": "avg", "question": "Return the average score in specimens.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT AVG(score) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT AVG(score) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "51f14088477a5f6be9c5_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "describe", "question": "Look up the columns and types for specimens.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "96a74866d51ea7899cd8_hi", "language": "hi", "backend": "supabase", "operation": "gte", "question": "specimens से वे पंक्तियाँ दिखाएँ जहाँ score 20 या उससे बड़ा है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE score >= 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE score >= 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1110104f12e10c5a007a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sum", "question": "i want the total amount of inspections", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT SUM(amount) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT SUM(amount) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "797213190eddac109b96_hi", "language": "hi", "backend": "supabase", "operation": "sql_error", "question": "पिछला क्वेरी विफल रहा क्योंकि एक कॉलम अज्ञात था। दोबारा प्रयास करने से पहले inspections स्कीमा की पुष्टि करें।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "28530a7607feca135b73_en", "language": "en", "backend": "supabase", "operation": "sum", "question": "Output the total price for the specimens entity.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT SUM(price) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT SUM(price) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "6b9bacf4d9aa380af183_hi", "language": "hi", "backend": "supabase", "operation": "ambiguous", "question": "reservations में उपलब्ध सर्वोत्तम रिकॉर्ड प्रिंट करें।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "924dbd9ad819c0a58b24_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sum", "question": "i want the total salary of inspections", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT SUM(salary) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT SUM(salary) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f7257c4a2eded61f5c86_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sort_desc", "question": "retrieve all records from specimens sorted by score high to low", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT * FROM specimens ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "4b73bea165fb5f9555f3_hi", "language": "hi", "backend": "supabase", "operation": "sql_error", "question": "पिछला क्वेरी विफल रहा क्योंकि एक कॉलम अज्ञात था। दोबारा प्रयास करने से पहले inspections स्कीमा की पुष्टि करें।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "82c104614df1ce6e2762_en", "language": "en", "backend": "supabase", "operation": "gt", "question": "Bring up the rows in specimens where salary is over 100.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE salary > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT * FROM specimens WHERE salary > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "40464b00088c9159e3d0_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "count", "question": "how many lines in inspections?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT COUNT(*) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT COUNT(*) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b564e1a7b1e7fc6d2ab4_hinglish", "language": "hinglish", "backend": "supabase", "operation": "avg", "question": "specimens me score ka average return karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT AVG(score) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT AVG(score) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0a2ec84ba0e594d94b9e_hi", "language": "hi", "backend": "supabase", "operation": "sql_error", "question": "पिछला क्वेरी विफल रहा क्योंकि एक कॉलम अज्ञात था। दोबारा प्रयास करने से पहले reservations स्कीमा की पुष्टि करें।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "bf7bc588f62f075ec1ca_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "count", "question": "how many lines in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT COUNT(*) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT COUNT(*) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5c6734bdfead38b51752_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gte", "question": "output records from specimens where quantity is not below 10000", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE quantity >= 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE quantity >= 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "ac6cc6f05e6d013971e5_hinglish", "language": "hinglish", "backend": "supabase", "operation": "max", "question": "specimens mein quantity ka sabse bada number kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT MAX(quantity) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT MAX(quantity) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "82c104614df1ce6e2762_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gt", "question": "show me the entries in specimens where salary is strictly greater than 100", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE salary > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE salary > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "40464b00088c9159e3d0_hinglish", "language": "hinglish", "backend": "supabase", "operation": "count", "question": "Batao inspections mein kitni entries hain.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT COUNT(*) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT COUNT(*) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "80ed46f6e275ed01c6c1_hi", "language": "hi", "backend": "supabase", "operation": "count_eq", "question": "exhibits में category Pune वाले रिकॉर्ड्स की संख्या दीजिए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM exhibits WHERE category = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits WHERE category = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1896aafc31bd0a7d24ec_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sum", "question": "i want the total balance of exhibits", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT SUM(balance) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT SUM(balance) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ed474bd8a459cbfaa5d2_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "year", "question": "Fetch specimens entries where recorded_at year is 2021.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE EXTRACT(YEAR FROM recorded_at) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT * FROM specimens WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "51f14088477a5f6be9c5_en", "language": "en", "backend": "supabase", "operation": "describe", "question": "Reveal the column structure and types for specimens.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "fea44ee93f1cea0883dd_hi", "language": "hi", "backend": "supabase", "operation": "count_eq", "question": "exhibits में category Delhi वाले रिकॉर्ड्स की संख्या दीजिए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM exhibits WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7dd163238936d96d0bd4_hi", "language": "hi", "backend": "supabase", "operation": "distinct", "question": "मुझे reservations में उपलब्ध city के विशिष्ट मान देखने हैं।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT city FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4fb8670235a435c82c88_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "describe", "question": "Look up the columns and types for exhibits.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_exhibits", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "864b7a7bf739071bb030_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "and", "question": "find rows in specimens where status is Pune and balance is above 0", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE status = 'Pune' AND balance > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE status = 'Pune' AND balance > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "1110104f12e10c5a007a_en", "language": "en", "backend": "supabase", "operation": "sum", "question": "Output the total amount for the inspections entity.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT SUM(amount) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT SUM(amount) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "37a16740c32af37d3b15_hi", "language": "hi", "backend": "supabase", "operation": "lte", "question": "reservations से वे पंक्तियां खोजें जहां score 50 से बड़ा नहीं है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE score <= 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score <= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bf7bc588f62f075ec1ca_hinglish", "language": "hinglish", "backend": "supabase", "operation": "count", "question": "Batao exhibits mein kitni entries hain.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT COUNT(*) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT COUNT(*) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "41f5fbcefb9d550068a0_hi", "language": "hi", "backend": "supabase", "operation": "missing_schema", "question": "कृपया specimens के लिए SQL लिखने से पहले उपलब्ध तालिकाओं का पता लगाएं, क्योंकि मैंने डेटाबेस स्कीमा नहीं दी है।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "924dbd9ad819c0a58b24_en", "language": "en", "backend": "supabase", "operation": "sum", "question": "Output the total salary for the inspections entity.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT SUM(salary) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT SUM(salary) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f7257c4a2eded61f5c86_en", "language": "en", "backend": "supabase", "operation": "sort_desc", "question": "Display the entire table specimens sorted by score from highest to lowest.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "7e1cd79fe94cad9c7a9e_en", "language": "en", "backend": "supabase", "operation": "count_eq", "question": "Provide the row count for specimens where region is Books.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT COUNT(*) FROM specimens WHERE region = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT COUNT(*) FROM specimens WHERE region = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "7e1cd79fe94cad9c7a9e_hinglish", "language": "hinglish", "backend": "supabase", "operation": "count_eq", "question": "Please count the rows in specimens where region is Books.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT COUNT(*) FROM specimens WHERE region = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT COUNT(*) FROM specimens WHERE region = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "35406fa024e66bcb5dd7_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lt", "question": "can you select the rows from specimens where score is less than 10000?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE score < 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT * FROM specimens WHERE score < 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "ed474bd8a459cbfaa5d2_en", "language": "en", "backend": "supabase", "operation": "year", "question": "Show me the specimens rows for 2021 using recorded_at.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE EXTRACT(YEAR FROM recorded_at) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "223765dc6e033f0ab2fd_en", "language": "en", "backend": "supabase", "operation": "month", "question": "Select entries from specimens where created_at corresponds to 9.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE EXTRACT(MONTH FROM created_at) = 9;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE EXTRACT(MONTH FROM created_at) = 9;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "51f14088477a5f6be9c5_hinglish", "language": "hinglish", "backend": "supabase", "operation": "describe", "question": "Mujhe specimens ke columns aur types ke baare mein info do.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "92e36c8a6ca6e7aa2298_hi", "language": "hi", "backend": "supabase", "operation": "group_count", "question": "reservations में हर category के लिए पंक्तियों की संख्या खोजें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "864b7a7bf739071bb030_en", "language": "en", "backend": "supabase", "operation": "and", "question": "Show me the specimens rows where status is Pune and balance exceeds 0.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE status = 'Pune' AND balance > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE status = 'Pune' AND balance > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "2ffce857234ca4c32ec9_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project", "question": "bas reservations ka title do please", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT title FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT title FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8fa63f2c53b1bc742d49_hinglish", "language": "hinglish", "backend": "supabase", "operation": "null", "question": "inspections se title null wali rows select karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "28530a7607feca135b73_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sum", "question": "specimens entity ke liye price ka total output do.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT SUM(price) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT SUM(price) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4dac1dcf2ef14bfee2d2_hi", "language": "hi", "backend": "supabase", "operation": "group_count", "question": "reservations में हर category के लिए पंक्तियों की संख्या खोजें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5c6734bdfead38b51752_en", "language": "en", "backend": "supabase", "operation": "gte", "question": "Show the data from specimens where quantity is 10000 or above.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE quantity >= 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE quantity >= 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "f7257c4a2eded61f5c86_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sort_desc", "question": "specimens ki poori table score highest se lowest order mein dikhao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "44135783de9bd091bfe5_hi", "language": "hi", "backend": "supabase", "operation": "missing_schema", "question": "कृपया inspections के लिए SQL लिखने से पहले उपलब्ध तालिकाओं का पता लगाएं, क्योंकि मैंने डेटाबेस स्कीमा नहीं दी है।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_inspections", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_inspections\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "28a7ce61b1233483bf73_hi", "language": "hi", "backend": "supabase", "operation": "sort_desc", "question": "specimens की पूरी सारणी salary के उच्चतम से निम्नतम क्रम में दिखाएँ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c4f25575abfd24e3cd63_hi", "language": "hi", "backend": "supabase", "operation": "lte", "question": "reservations से वे पंक्तियां खोजें जहां cost 1 से बड़ा नहीं है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE cost <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8b1939fffee21884f667_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "null", "question": "Show me the rows from specimens where item_name is null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "8b1939fffee21884f667_hinglish", "language": "hinglish", "backend": "supabase", "operation": "null", "question": "specimens se item_name null wali rows select karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1896aafc31bd0a7d24ec_en", "language": "en", "backend": "supabase", "operation": "sum", "question": "Output the total balance for the exhibits entity.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT SUM(balance) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT SUM(balance) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ed474bd8a459cbfaa5d2_hinglish", "language": "hinglish", "backend": "supabase", "operation": "year", "question": "specimens mein recorded_at mein 2021 wale records dikhao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE EXTRACT(YEAR FROM recorded_at) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "223765dc6e033f0ab2fd_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "month", "question": "extract rows from specimens where created_at month is 9", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE EXTRACT(MONTH FROM created_at) = 9;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE EXTRACT(MONTH FROM created_at) = 9;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "6db4ac493194a52cbd3a_hi", "language": "hi", "backend": "supabase", "operation": "missing_schema", "question": "कृपया reservations के लिए SQL लिखने से पहले उपलब्ध तालिकाओं का पता लगाएं, क्योंकि मैंने डेटाबेस स्कीमा नहीं दी है।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_reservations", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_reservations\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "fd0bab1cc08d7298f131_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "top", "question": "I need 20 rows from exhibits with salary sorted descending.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4fb8670235a435c82c88_en", "language": "en", "backend": "supabase", "operation": "describe", "question": "Reveal the column structure and types for exhibits.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_exhibits", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "1110104f12e10c5a007a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sum", "question": "inspections entity ke liye amount ka total output do.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT SUM(amount) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT SUM(amount) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2ffce857234ca4c32ec9_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project", "question": "display title from reservations and nothing else", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT title FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT title FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8fa63f2c53b1bc742d49_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "null", "question": "Show me the rows from inspections where title is null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "56189c5cdc014dd771be_hi", "language": "hi", "backend": "supabase", "operation": "lt", "question": "reservations से उन पंक्तियों का क्वेरी करें जहाँ balance 50 से कम है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE balance < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3eea55cd4532636ddffe_en", "language": "en", "backend": "supabase", "operation": "not_null", "question": "Present the data from specimens where display_name is not null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE display_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE display_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "3eea55cd4532636ddffe_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "not_null", "question": "Find the rows in specimens where display_name is not null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE display_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE display_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "924dbd9ad819c0a58b24_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sum", "question": "inspections entity ke liye salary ka total output do.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT SUM(salary) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT SUM(salary) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "82c104614df1ce6e2762_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gt", "question": "specimens se un rows ko display karo jahan salary 100 se more hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE salary > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE salary > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "35406fa024e66bcb5dd7_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lt", "question": "specimens se un rows ko query karo jahan score 10000 se kam hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE score < 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE score < 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7094d70d08791284b632_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "min", "question": "What is the floor salary in reservations?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT MIN(salary) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT MIN(salary) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b30330bb2c17b32c6883_hi", "language": "hi", "backend": "supabase", "operation": "sort_asc", "question": "exhibits से सभी डेटा cost के बढ़ते हुए क्रम में लें और दिखाएं।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY cost ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4a79981adf7647bac6b9_en", "language": "en", "backend": "supabase", "operation": "gt", "question": "Bring up the rows in exhibits where amount is over 500.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE amount > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE amount > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4fb8670235a435c82c88_hinglish", "language": "hinglish", "backend": "supabase", "operation": "describe", "question": "Mujhe exhibits ke columns aur types ke baare mein info do.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_exhibits", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "ae26d8175253b650bb6b_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "missing_value", "question": "execute filter on specimens for region, but i have not specified a value", "expected": {"action": "clarify", "question": "Which value should region equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should region equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "e8cfdfba296bc283222a_en", "language": "en", "backend": "supabase", "operation": "month", "question": "Select entries from exhibits where start_date corresponds to 7.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE EXTRACT(MONTH FROM start_date) = 7;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE EXTRACT(MONTH FROM start_date) = 7;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "71349134f8ae16809501_hi", "language": "hi", "backend": "supabase", "operation": "sort_desc", "question": "exhibits की पूरी सारणी salary के उच्चतम से निम्नतम क्रम में दिखाएँ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5c6734bdfead38b51752_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gte", "question": "specimens se rows dikhao jahan quantity 10000 ya usse bada hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE quantity >= 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE quantity >= 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "aeeb76897a32b9fd2df4_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "min", "question": "What is the floor amount in reservations?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT MIN(amount) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT MIN(amount) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "785d62b3d5e4932f5c33_en", "language": "en", "backend": "supabase", "operation": "bottom", "question": "Present the 20 minimal rating entries from inspections in ascending order.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections ORDER BY rating ASC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections ORDER BY rating ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "785d62b3d5e4932f5c33_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "bottom", "question": "print the 20 minimal rating rows in inspections from lowest up", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections ORDER BY rating ASC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections ORDER BY rating ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0b8689cf8e381cbcfe86_hi", "language": "hi", "backend": "supabase", "operation": "and", "question": "reservations में वे पंक्तियाँ खोजें जहाँ department pending है और price 10000 से ऊपर है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE department = 'pending' AND price > 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE department = 'pending' AND price > 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7094d70d08791284b632_hinglish", "language": "hinglish", "backend": "supabase", "operation": "min", "question": "reservations mein salary ka minimum reading kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT MIN(salary) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT MIN(salary) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8b1939fffee21884f667_en", "language": "en", "backend": "supabase", "operation": "null", "question": "I want to view the rows in specimens where item_name is null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "1896aafc31bd0a7d24ec_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sum", "question": "exhibits entity ke liye balance ka total output do.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT SUM(balance) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT SUM(balance) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b564e1a7b1e7fc6d2ab4_hi", "language": "hi", "backend": "supabase", "operation": "avg", "question": "specimens में score का औसत लौटाएँ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT AVG(score) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT AVG(score) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e92015bce168f8f9a90a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sort_desc", "question": "retrieve all records from exhibits sorted by score high to low", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "223765dc6e033f0ab2fd_hinglish", "language": "hinglish", "backend": "supabase", "operation": "month", "question": "Show me the data from specimens where created_at ka month 9 hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE EXTRACT(MONTH FROM created_at) = 9;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE EXTRACT(MONTH FROM created_at) = 9;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "4a79981adf7647bac6b9_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gt", "question": "show me the entries in exhibits where amount is strictly greater than 500", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE amount > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE amount > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "61b3db48f1ff12dc69e1_en", "language": "en", "backend": "supabase", "operation": "group_sum", "question": "Please provide the total quantity for each kind in exhibits.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT kind, SUM(quantity) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT kind, SUM(quantity) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e8cfdfba296bc283222a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "month", "question": "extract rows from exhibits where start_date month is 7", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE EXTRACT(MONTH FROM start_date) = 7;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE EXTRACT(MONTH FROM start_date) = 7;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8fa63f2c53b1bc742d49_en", "language": "en", "backend": "supabase", "operation": "null", "question": "I want to view the rows in inspections where title is null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "393d51bc9e30d68b474e_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "and", "question": "find rows in inspections where status is active and cost is above 5", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE status = 'active' AND cost > 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE status = 'active' AND cost > 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3eea55cd4532636ddffe_hinglish", "language": "hinglish", "backend": "supabase", "operation": "not_null", "question": "specimens ka data present karo jahan display_name null nahi hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE display_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE display_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0ce22f4c8bc7554d586d_hi", "language": "hi", "backend": "supabase", "operation": "lt", "question": "reservations से उन पंक्तियों का क्वेरी करें जहाँ score 50 से कम है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE score < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5ce4cc7ea9d93a0aa4c5_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sum", "question": "i want the total quantity of reservations", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT SUM(quantity) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT SUM(quantity) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "aeeb76897a32b9fd2df4_hinglish", "language": "hinglish", "backend": "supabase", "operation": "min", "question": "reservations mein amount ka minimum reading kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT MIN(amount) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT MIN(amount) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "475287613764159ee5eb_en", "language": "en", "backend": "supabase", "operation": "group_count", "question": "Find the count of rows per category in reservations.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "475287613764159ee5eb_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "group_count", "question": "give me the count for each category rows in reservations", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fd0bab1cc08d7298f131_hinglish", "language": "hinglish", "backend": "supabase", "operation": "top", "question": "Top 20 rows exhibits se lo, salary ke hisaab se highest first.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "61b3db48f1ff12dc69e1_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "group_sum", "question": "I need to see the total quantity for each kind in exhibits", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT kind, SUM(quantity) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT kind, SUM(quantity) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c86deada03a046468575_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project", "question": "bas reservations ka item_name do please", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "864b7a7bf739071bb030_hinglish", "language": "hinglish", "backend": "supabase", "operation": "and", "question": "specimens mein wo rows dhundo jahan status Pune hai aur balance 0 se upar hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE status = 'Pune' AND balance > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE status = 'Pune' AND balance > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2ffce857234ca4c32ec9_en", "language": "en", "backend": "supabase", "operation": "project", "question": "Display title from reservations as the sole result.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT title FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT title FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7d18561ec942f18f1299_hi", "language": "hi", "backend": "supabase", "operation": "count_eq", "question": "reservations में status O'Reilly वाले रिकॉर्ड्स की संख्या दीजिए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM reservations WHERE status = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM reservations WHERE status = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "393d51bc9e30d68b474e_en", "language": "en", "backend": "supabase", "operation": "and", "question": "Show me the inspections rows where status is active and cost exceeds 5.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE status = 'active' AND cost > 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE status = 'active' AND cost > 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7a33de7fd7159808b04a_en", "language": "en", "backend": "supabase", "operation": "month", "question": "Select entries from inspections where created_at corresponds to 11.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE EXTRACT(MONTH FROM created_at) = 11;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE EXTRACT(MONTH FROM created_at) = 11;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e02d95c6aba4ed336a6a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gte", "question": "output records from reservations where cost is not below 50", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE cost >= 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE cost >= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "40464b00088c9159e3d0_en", "language": "en", "backend": "supabase", "operation": "count", "question": "What is the volume of data rows in inspections?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT COUNT(*) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT COUNT(*) FROM inspections WHERE department = 'Agra';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "7094d70d08791284b632_en", "language": "en", "backend": "supabase", "operation": "min", "question": "What is the baseline salary value in reservations?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT MIN(salary) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT MIN(salary) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "475287613764159ee5eb_hinglish", "language": "hinglish", "backend": "supabase", "operation": "group_count", "question": "reservations me pratyek category ke liye rows ki sankhya dhundo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1695d78b68e4d8fca6f0_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "and", "question": "find rows in exhibits where region is Delhi and amount is above 1500", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE region = 'Delhi' AND amount > 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE region = 'Delhi' AND amount > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e92015bce168f8f9a90a_en", "language": "en", "backend": "supabase", "operation": "sort_desc", "question": "Display the entire table exhibits sorted by score from highest to lowest.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e92015bce168f8f9a90a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sort_desc", "question": "exhibits ki poori table score highest se lowest order mein dikhao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4a79981adf7647bac6b9_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gt", "question": "exhibits se un rows ko display karo jahan amount 500 se more hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE amount > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE amount > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "645fd36c186709e64922_hi", "language": "hi", "backend": "supabase", "operation": "between", "question": "specimens से वे सभी आइटम चुनें जहाँ amount कॉलम का मान 1000 और 1010 के बीच है, समावेशी।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE amount BETWEEN 1000 AND 1010;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "'query'"}}
{"id": "61b3db48f1ff12dc69e1_hinglish", "language": "hinglish", "backend": "supabase", "operation": "group_sum", "question": "exhibits me kind ke hisaab se quantity ka total do please.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT kind, SUM(quantity) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT kind, SUM(quantity) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9ebca064e63b0347236b_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "having", "question": "Tell me the region groups in specimens with more than 5 rows and their count values.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "9ebca064e63b0347236b_hinglish", "language": "hinglish", "backend": "supabase", "operation": "having", "question": "Please specimens me 5 se zyada rows wale region groups aur unki count ko display karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "ae26d8175253b650bb6b_en", "language": "en", "backend": "supabase", "operation": "missing_value", "question": "Execute a filter on specimens based on region, but I have not specified a value.", "expected": {"action": "clarify", "question": "Which value should region equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should region equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "c86deada03a046468575_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project", "question": "display item_name from reservations and nothing else", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b7703ebecadd9433c5f8_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "top", "question": "I need 3 rows from reservations with quantity sorted descending.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations ORDER BY quantity DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations ORDER BY quantity DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e8cfdfba296bc283222a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "month", "question": "Show me the data from exhibits where start_date ka month 7 hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE EXTRACT(MONTH FROM start_date) = 7;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE EXTRACT(MONTH FROM start_date) = 7;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bf7bc588f62f075ec1ca_en", "language": "en", "backend": "supabase", "operation": "count", "question": "What is the volume of data rows in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT COUNT(*) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT COUNT(*) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bd5fbea0dbbb3414c53c_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "having", "question": "Tell me the region groups in specimens with more than 0 rows and their count values.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "bd5fbea0dbbb3414c53c_hinglish", "language": "hinglish", "backend": "supabase", "operation": "having", "question": "Please specimens me 0 se zyada rows wale region groups aur unki count ko display karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "9bb888f912ca90f30c9a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "top", "question": "I need 20 rows from reservations with rating sorted descending.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations ORDER BY rating DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations ORDER BY rating DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a7efbaf306821c7ab040_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "year", "question": "Fetch reservations entries where order_date year is 2023.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE EXTRACT(YEAR FROM order_date) = 2023;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM order_date) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a7efbaf306821c7ab040_hinglish", "language": "hinglish", "backend": "supabase", "operation": "year", "question": "reservations mein order_date mein 2023 wale records dikhao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE EXTRACT(YEAR FROM order_date) = 2023;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM order_date) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7a33de7fd7159808b04a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "month", "question": "extract rows from inspections where created_at month is 11", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE EXTRACT(MONTH FROM created_at) = 11;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE EXTRACT(MONTH FROM created_at) = 11;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5ce4cc7ea9d93a0aa4c5_en", "language": "en", "backend": "supabase", "operation": "sum", "question": "Output the total quantity for the reservations entity.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT SUM(quantity) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT SUM(quantity) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "aeeb76897a32b9fd2df4_en", "language": "en", "backend": "supabase", "operation": "min", "question": "What is the baseline amount value in reservations?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT MIN(amount) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT MIN(amount) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7d4234476674a04380e6_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gte", "question": "output records from reservations where score is not below 1500", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE score >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE score >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1695d78b68e4d8fca6f0_en", "language": "en", "backend": "supabase", "operation": "and", "question": "Show me the exhibits rows where region is Delhi and amount exceeds 1500.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE region = 'Delhi' AND amount > 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE region = 'Delhi' AND amount > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fd0bab1cc08d7298f131_en", "language": "en", "backend": "supabase", "operation": "top", "question": "Give me the top 20 rows of exhibits sorted by salary from high to low.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "039280d3483e998ccd1b_en", "language": "en", "backend": "supabase", "operation": "group_count", "question": "Find the count of rows per department in reservations.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT department, COUNT(*) FROM reservations GROUP BY department;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "039280d3483e998ccd1b_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "group_count", "question": "give me the count for each department rows in reservations", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT department, COUNT(*) FROM reservations GROUP BY department;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "523a83d6ca07c1a1334a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "missing_value", "question": "execute filter on exhibits for department, but i have not specified a value", "expected": {"action": "clarify", "question": "Which value should department equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should department equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "5d05a2e11d5abab8423e_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "missing_value", "question": "execute filter on inspections for region, but i have not specified a value", "expected": {"action": "clarify", "question": "Which value should region equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should region equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "f4fb7fafb95b5c8287f5_en", "language": "en", "backend": "supabase", "operation": "group_count", "question": "Find the count of rows per department in reservations.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT department, COUNT(*) FROM reservations GROUP BY department;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f4fb7fafb95b5c8287f5_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "group_count", "question": "give me the count for each department rows in reservations", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT department, COUNT(*) FROM reservations GROUP BY department;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a7efbaf306821c7ab040_en", "language": "en", "backend": "supabase", "operation": "year", "question": "Show me the reservations rows for 2023 using order_date.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE EXTRACT(YEAR FROM order_date) = 2023;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM order_date) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e02d95c6aba4ed336a6a_en", "language": "en", "backend": "supabase", "operation": "gte", "question": "Show the data from reservations where cost is 50 or above.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE cost >= 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE cost >= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e02d95c6aba4ed336a6a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gte", "question": "reservations se rows dikhao jahan cost 50 ya usse bada hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE cost >= 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE cost >= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5ce4cc7ea9d93a0aa4c5_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sum", "question": "reservations entity ke liye quantity ka total output do.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT SUM(quantity) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT SUM(quantity) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "039280d3483e998ccd1b_hinglish", "language": "hinglish", "backend": "supabase", "operation": "group_count", "question": "reservations me pratyek department ke liye rows ki sankhya dhundo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT department, COUNT(*) FROM reservations GROUP BY department;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "393d51bc9e30d68b474e_hinglish", "language": "hinglish", "backend": "supabase", "operation": "and", "question": "inspections mein wo rows dhundo jahan status active hai aur cost 5 se upar hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE status = 'active' AND cost > 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE status = 'active' AND cost > 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f4fb7fafb95b5c8287f5_hinglish", "language": "hinglish", "backend": "supabase", "operation": "group_count", "question": "reservations me pratyek department ke liye rows ki sankhya dhundo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT department, COUNT(*) FROM reservations GROUP BY department;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7a33de7fd7159808b04a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "month", "question": "Show me the data from inspections where created_at ka month 11 hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE EXTRACT(MONTH FROM created_at) = 11;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE EXTRACT(MONTH FROM created_at) = 11;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ac6cc6f05e6d013971e5_hi", "language": "hi", "backend": "supabase", "operation": "max", "question": "specimens में quantity का सबसे बड़ा नंबर क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT MAX(quantity) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT MAX(quantity) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1a1af4deb475f0224a59_hi", "language": "hi", "backend": "supabase", "operation": "bottom", "question": "price के बढ़ते क्रम में inspections की 10 सबसे कम पंक्तियां, सबसे कम से शुरू होकर, लें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections ORDER BY price ASC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY price ASC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1d82ef2b9527e797c466_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "date_after", "question": "Query rows in inspections where order_date is 2022-04-15 onwards.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE order_date >= '2022-04-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE order_date >= '2022-04-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7d4234476674a04380e6_en", "language": "en", "backend": "supabase", "operation": "gte", "question": "Show the data from reservations where score is 1500 or above.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE score >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE score >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7d4234476674a04380e6_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gte", "question": "reservations se rows dikhao jahan score 1500 ya usse bada hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE score >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE score >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c86deada03a046468575_en", "language": "en", "backend": "supabase", "operation": "project", "question": "Display item_name from reservations as the sole result.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b7703ebecadd9433c5f8_hinglish", "language": "hinglish", "backend": "supabase", "operation": "top", "question": "Top 3 rows reservations se lo, quantity ke hisaab se highest first.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations ORDER BY quantity DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations ORDER BY quantity DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f1f8414c2d47f5396d69_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "date_after", "question": "Query rows in exhibits where created_at is 2023-03-15 onwards.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE created_at >= '2023-03-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE created_at >= '2023-03-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9bb888f912ca90f30c9a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "top", "question": "Top 20 rows reservations se lo, rating ke hisaab se highest first.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations ORDER BY rating DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations ORDER BY rating DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dd6cdefd15edcdcddb76_hi", "language": "hi", "backend": "supabase", "operation": "bottom", "question": "salary के बढ़ते क्रम में exhibits की 5 सबसे कम पंक्तियां, सबसे कम से शुरू होकर, लें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY salary ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dddd09f34c0e849282a8_en", "language": "en", "backend": "supabase", "operation": "gt", "question": "Bring up the rows in reservations where salary is over 20.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE salary > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE salary > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f35f7a78a14e903920ee_hi", "language": "hi", "backend": "supabase", "operation": "month", "question": "मुझे reservations से वे डेटा पंक्तियां दें जहाँ order_date का महीना 3 है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE EXTRACT(MONTH FROM order_date) = 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(MONTH FROM order_date) = 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1695d78b68e4d8fca6f0_hinglish", "language": "hinglish", "backend": "supabase", "operation": "and", "question": "exhibits mein wo rows dhundo jahan region Delhi hai aur amount 1500 se upar hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE region = 'Delhi' AND amount > 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE region = 'Delhi' AND amount > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c07bcd68edd3354fcd83_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "all", "question": "fetch all rows from specimens", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ae26d8175253b650bb6b_hinglish", "language": "hinglish", "backend": "supabase", "operation": "missing_value", "question": "region ke basis pe specimens pe ek filter execute karo, lekin maine koi value specify nahi ki hai.", "expected": {"action": "clarify", "question": "region ki kaunsi value chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"region ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "375b86b5cd0d00cd7868_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "having", "question": "Tell me the department groups in exhibits with more than 2 rows and their count values.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "375b86b5cd0d00cd7868_hinglish", "language": "hinglish", "backend": "supabase", "operation": "having", "question": "Please exhibits me 2 se zyada rows wale department groups aur unki count ko display karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "523a83d6ca07c1a1334a_en", "language": "en", "backend": "supabase", "operation": "missing_value", "question": "Execute a filter on exhibits based on department, but I have not specified a value.", "expected": {"action": "clarify", "question": "Which value should department equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should department equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "db7eb47a2bad512e837d_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "all", "question": "fetch all rows from specimens", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2ffce857234ca4c32ec9_hi", "language": "hi", "backend": "supabase", "operation": "project", "question": "सिर्फ reservations का title दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT title FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT title FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ccbb8ac5e43114f43047_en", "language": "en", "backend": "supabase", "operation": "between", "question": "Select all items from exhibits where the balance column has a value between 50 and 100, inclusive.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE balance BETWEEN 50 AND 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE balance BETWEEN 50 AND 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "28530a7607feca135b73_hi", "language": "hi", "backend": "supabase", "operation": "sum", "question": "specimens में price के सभी मानों का योग करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT SUM(price) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT SUM(price) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5d05a2e11d5abab8423e_en", "language": "en", "backend": "supabase", "operation": "missing_value", "question": "Execute a filter on inspections based on region, but I have not specified a value.", "expected": {"action": "clarify", "question": "Which value should region equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should region equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "6dd9f03c26298630777f_en", "language": "en", "backend": "supabase", "operation": "between", "question": "Select all items from inspections where the score column has a value between 1 and 101, inclusive.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE score BETWEEN 1 AND 101;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE score BETWEEN 1 AND 101;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6c37ae504c7cb31a6ef3_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "list_tables", "question": "tell me the tables in this database", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "dddd09f34c0e849282a8_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gt", "question": "show me the entries in reservations where salary is strictly greater than 20", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE salary > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE salary > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a4cdc86496ddd0c1cc90_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "avg", "question": "determine the mean cost in specimens", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT AVG(cost) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT AVG(cost) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "785d62b3d5e4932f5c33_hinglish", "language": "hinglish", "backend": "supabase", "operation": "bottom", "question": "rating ke badhte order me inspections ki 20 sabse kam rows, lowest se shuru karke, lao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections ORDER BY rating ASC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections ORDER BY rating ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "23c0aa3a4596f995ad2a_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count", "question": "how many lines in specimens?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "58eb6ab2fb9c69becdbd_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "date_after", "question": "Query rows in reservations where created_at is 2022-08-15 onwards.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE created_at >= '2022-08-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE created_at >= '2022-08-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c07bcd68edd3354fcd83_en", "language": "en", "backend": "mysql", "operation": "all", "question": "Get all rows from specimens.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9ebca064e63b0347236b_en", "language": "en", "backend": "supabase", "operation": "having", "question": "Give me the region groups in specimens where the row count is higher than 5, along with the count values.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "e3d12f1bf51bbf54a331_hi", "language": "hi", "backend": "supabase", "operation": "contains", "question": "specimens से वे पंक्तियाँ दिखाएं जिनमें description North के समान है, मामला अंतर नजरअंदाज करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE LOWER(description) LIKE LOWER('%North%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(description) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "55cd623d82474692b1a9_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count", "question": "how many lines in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1110104f12e10c5a007a_hi", "language": "hi", "backend": "supabase", "operation": "sum", "question": "inspections में amount के सभी मानों का योग करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT SUM(amount) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT SUM(amount) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b7703ebecadd9433c5f8_en", "language": "en", "backend": "supabase", "operation": "top", "question": "Give me the top 3 rows of reservations sorted by quantity from high to low.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations ORDER BY quantity DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations ORDER BY quantity DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "db7eb47a2bad512e837d_en", "language": "en", "backend": "mysql", "operation": "all", "question": "Get all rows from specimens.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bd5fbea0dbbb3414c53c_en", "language": "en", "backend": "supabase", "operation": "having", "question": "Give me the region groups in specimens where the row count is higher than 0, along with the count values.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "e6380b8531b4b3eb7e38_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project", "question": "bas specimens ka name do please", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "924dbd9ad819c0a58b24_hi", "language": "hi", "backend": "supabase", "operation": "sum", "question": "inspections में salary के सभी मानों का योग करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT SUM(salary) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT SUM(salary) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9bb888f912ca90f30c9a_en", "language": "en", "backend": "supabase", "operation": "top", "question": "Give me the top 20 rows of reservations sorted by rating from high to low.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations ORDER BY rating DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations ORDER BY rating DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6c37ae504c7cb31a6ef3_en", "language": "en", "backend": "mysql", "operation": "list_tables", "question": "Display the schema tables in this database.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6c37ae504c7cb31a6ef3_hinglish", "language": "hinglish", "backend": "mysql", "operation": "list_tables", "question": "Is database ki tables ka detail do.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "dddd09f34c0e849282a8_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gt", "question": "reservations se un rows ko display karo jahan salary 20 se more hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE salary > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE salary > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "40464b00088c9159e3d0_hi", "language": "hi", "backend": "supabase", "operation": "count", "question": "inspections में कितनी लाइनें मौजूद हैं?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT COUNT(*) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT COUNT(*) FROM inspections WHERE department = 'Agra';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "8efcf517aee64807a8e5_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "avg", "question": "determine the mean balance in specimens", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "797b9b914530f2eb502a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "final", "question": "db tool returned 48 rows. tell me the total.", "expected": {"action": "answer", "text": "The tool returned 48 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 48 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "797b9b914530f2eb502a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "final", "question": "Kitni rows aayin database tool se? Count hai 48.", "expected": {"action": "answer", "text": "Tool ne 48 rows return ki."}, "output": "{\"action\":\"answer\",\"text\":\"Tool ne 48 rows return ki.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "23c0aa3a4596f995ad2a_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count", "question": "Batao specimens mein kitni entries hain.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "49c2e66e48d062e5c69c_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "write", "question": "Remove all records from specimens now.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "64372a29664a7eaddbe2_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "avg", "question": "determine the mean amount in specimens", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT AVG(amount) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT AVG(amount) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "55cd623d82474692b1a9_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count", "question": "Batao specimens mein kitni entries hain.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e02df083b1ff1b7fbcf8_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "join_filter", "question": "show me exhibits.customer_name for exhibits entries where exhibits.group_id links to exhibits_groups.id and exhibits_groups.label is North", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT exhibits.customer_name FROM exhibits JOIN exhibits_groups ON exhibits.group_id = exhibits_groups.id WHERE exhibits_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT exhibits.customer_name FROM exhibits JOIN exhibits_groups ON exhibits.group_id = exhibits_groups.id WHERE exhibits_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e02df083b1ff1b7fbcf8_hinglish", "language": "hinglish", "backend": "supabase", "operation": "join_filter", "question": "Please exhibits.customer_name dikhao jahan exhibits exhibits.group_id ke through exhibits_groups se juda hai aur exhibits_groups.label North hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT exhibits.customer_name FROM exhibits JOIN exhibits_groups ON exhibits.group_id = exhibits_groups.id WHERE exhibits_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT exhibits.customer_name FROM exhibits JOIN exhibits_groups ON exhibits.group_id = exhibits_groups.id WHERE exhibits_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2ca6667f987742cde1e7_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "write", "question": "Remove all records from specimens now.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "ccbb8ac5e43114f43047_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "between", "question": "pull the rows from exhibits where balance is greater than or equal to 50 and less than or equal to 100", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE balance BETWEEN 50 AND 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE balance BETWEEN 50 AND 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bf7bc588f62f075ec1ca_hi", "language": "hi", "backend": "supabase", "operation": "count", "question": "exhibits में कितनी लाइनें मौजूद हैं?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT COUNT(*) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT COUNT(*) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9bf8948becf74e81a103_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "list_tables", "question": "tell me the tables in this database", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "e6380b8531b4b3eb7e38_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project", "question": "display name from specimens and nothing else", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6dd9f03c26298630777f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "between", "question": "pull the rows from inspections where score is greater than or equal to 1 and less than or equal to 101", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE score BETWEEN 1 AND 101;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE score BETWEEN 1 AND 101;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a4cdc86496ddd0c1cc90_en", "language": "en", "backend": "mysql", "operation": "avg", "question": "Return the average cost in specimens.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT AVG(cost) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT AVG(cost) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1d82ef2b9527e797c466_en", "language": "en", "backend": "supabase", "operation": "date_after", "question": "Extract rows from inspections where order_date falls on 2022-04-15 or beyond.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE order_date >= '2022-04-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE order_date >= '2022-04-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c4210296305105ab7472_en", "language": "en", "backend": "mysql", "operation": "ambiguous", "question": "Print the best available records in specimens.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "c4210296305105ab7472_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "ambiguous", "question": "Give me the top records in specimens.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "1896aafc31bd0a7d24ec_hi", "language": "hi", "backend": "supabase", "operation": "sum", "question": "exhibits में balance के सभी मानों का योग करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT SUM(balance) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT SUM(balance) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8a3194b9f492b50b8100_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "distinct", "question": "i need the distinct category values from specimens", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT category FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d61a27de98bb7d44f15d_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count", "question": "how many lines in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3b3a8e441f0ae5cbaad4_en", "language": "en", "backend": "mysql", "operation": "ambiguous", "question": "Print the best available records in specimens.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "3b3a8e441f0ae5cbaad4_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "ambiguous", "question": "Give me the top records in specimens.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "69ac9800bd7a3ee23ccc_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "distinct", "question": "i need the distinct category values from specimens", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT category FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f1f8414c2d47f5396d69_en", "language": "en", "backend": "supabase", "operation": "date_after", "question": "Extract rows from exhibits where created_at falls on 2023-03-15 or beyond.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE created_at >= '2023-03-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE created_at >= '2023-03-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e02df083b1ff1b7fbcf8_en", "language": "en", "backend": "supabase", "operation": "join_filter", "question": "Give me exhibits.customer_name for exhibits entries where exhibits.group_id points to exhibits_groups.id and exhibits_groups.label is North.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT exhibits.customer_name FROM exhibits JOIN exhibits_groups ON exhibits.group_id = exhibits_groups.id WHERE exhibits_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT exhibits.customer_name FROM exhibits JOIN exhibits_groups ON exhibits.group_id = exhibits_groups.id WHERE exhibits_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ccbb8ac5e43114f43047_hinglish", "language": "hinglish", "backend": "supabase", "operation": "between", "question": "exhibits se wo saare items select karo jahan balance column ka value 50 aur 100 ke beech hai, inclusive.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE balance BETWEEN 50 AND 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE balance BETWEEN 50 AND 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8fa63f2c53b1bc742d49_hi", "language": "hi", "backend": "supabase", "operation": "null", "question": "inspections से title null वाली पंक्तियाँ चुनें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9bf8948becf74e81a103_en", "language": "en", "backend": "mysql", "operation": "list_tables", "question": "Display the schema tables in this database.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "9bf8948becf74e81a103_hinglish", "language": "hinglish", "backend": "mysql", "operation": "list_tables", "question": "Is database ki tables ka detail do.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6dd9f03c26298630777f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "between", "question": "inspections se wo saare items select karo jahan score column ka value 1 aur 101 ke beech hai, inclusive.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE score BETWEEN 1 AND 101;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE score BETWEEN 1 AND 101;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1523073b9b419fc60ded_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count_eq", "question": "how many specimens rows where city is Books", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT COUNT(*) FROM specimens WHERE city = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens WHERE city = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a4cdc86496ddd0c1cc90_hinglish", "language": "hinglish", "backend": "mysql", "operation": "avg", "question": "specimens me cost ka average return karo.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT AVG(cost) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT AVG(cost) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "41ea5651dda516a4ac30_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "date_after", "question": "Query rows in reservations where order_date is 2023-12-15 onwards.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE order_date >= '2023-12-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE order_date >= '2023-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "adff4eabe2639c0c128c_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "all", "question": "fetch all rows from inspections", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8efcf517aee64807a8e5_en", "language": "en", "backend": "mysql", "operation": "avg", "question": "Return the average balance in specimens.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8b1939fffee21884f667_hi", "language": "hi", "backend": "supabase", "operation": "null", "question": "specimens से item_name null वाली पंक्तियाँ चुनें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "51f14088477a5f6be9c5_hi", "language": "hi", "backend": "supabase", "operation": "describe", "question": "टेबल specimens के कॉलम संरचना और टाइप्स दिखाइए।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "c07bcd68edd3354fcd83_hinglish", "language": "hinglish", "backend": "mysql", "operation": "all", "question": "specimens ke saare records get karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d61a27de98bb7d44f15d_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count", "question": "Batao specimens mein kitni entries hain.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "64372a29664a7eaddbe2_en", "language": "en", "backend": "mysql", "operation": "avg", "question": "Return the average amount in specimens.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT AVG(amount) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT AVG(amount) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6d9e81a9f52b8d33060a_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "all", "question": "fetch all rows from exhibits", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c86deada03a046468575_hi", "language": "hi", "backend": "supabase", "operation": "project", "question": "सिर्फ reservations का item_name दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "940596942f7c5e4bf758_hi", "language": "hi", "backend": "supabase", "operation": "list_tables", "question": "इस डेटाबेस की तालिकाओं का विवरण दीजिए।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "523a83d6ca07c1a1334a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "missing_value", "question": "department ke basis pe exhibits pe ek filter execute karo, lekin maine koi value specify nahi ki hai.", "expected": {"action": "clarify", "question": "department ki kaunsi value chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"department ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "db7eb47a2bad512e837d_hinglish", "language": "hinglish", "backend": "mysql", "operation": "all", "question": "specimens ke saare records get karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5d05a2e11d5abab8423e_hinglish", "language": "hinglish", "backend": "supabase", "operation": "missing_value", "question": "region ke basis pe inspections pe ek filter execute karo, lekin maine koi value specify nahi ki hai.", "expected": {"action": "clarify", "question": "region ki kaunsi value chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"region ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "3eea55cd4532636ddffe_hi", "language": "hi", "backend": "supabase", "operation": "not_null", "question": "specimens का डेटा दिखाएं जहाँ display_name null नहीं है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE display_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE display_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "adff4eabe2639c0c128c_en", "language": "en", "backend": "mysql", "operation": "all", "question": "Get all rows from inspections.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8efcf517aee64807a8e5_hinglish", "language": "hinglish", "backend": "mysql", "operation": "avg", "question": "specimens me balance ka average return karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "cd5bb3382b9e6c8604e8_en", "language": "en", "backend": "mysql", "operation": "max", "question": "What is the largest number for rating in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(rating) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(rating) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "cd5bb3382b9e6c8604e8_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "max", "question": "what is the upper bound of rating in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(rating) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(rating) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c4210296305105ab7472_hinglish", "language": "hinglish", "backend": "mysql", "operation": "ambiguous", "question": "specimens mein available best records print karo.", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "797b9b914530f2eb502a_en", "language": "en", "backend": "supabase", "operation": "final", "question": "The database tool returned 48 rows. Can you tell me the number?", "expected": {"action": "answer", "text": "The tool returned 48 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 48 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "9aed1992a793fbf97601_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "min", "question": "What is the floor price in exhibits?", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT MIN(price) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT MIN(price) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "58eb6ab2fb9c69becdbd_en", "language": "en", "backend": "supabase", "operation": "date_after", "question": "Extract rows from reservations where created_at falls on 2022-08-15 or beyond.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE created_at >= '2022-08-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE created_at >= '2022-08-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "49c2e66e48d062e5c69c_en", "language": "en", "backend": "mysql", "operation": "write", "question": "Expunge every row from specimens.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}}
{"id": "64372a29664a7eaddbe2_hinglish", "language": "hinglish", "backend": "mysql", "operation": "avg", "question": "specimens me amount ka average return karo.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT AVG(amount) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT AVG(amount) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3b3a8e441f0ae5cbaad4_hinglish", "language": "hinglish", "backend": "mysql", "operation": "ambiguous", "question": "specimens mein available best records print karo.", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "6d9e81a9f52b8d33060a_en", "language": "en", "backend": "mysql", "operation": "all", "question": "Get all rows from exhibits.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e2b48c4ec5252934cb20_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count_eq", "question": "how many inspections rows where city is pending", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT COUNT(*) FROM inspections WHERE city = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM inspections WHERE city = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "375b86b5cd0d00cd7868_en", "language": "en", "backend": "supabase", "operation": "having", "question": "Give me the department groups in exhibits where the row count is higher than 2, along with the count values.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4745f53221e07d4f5ae1_hi", "language": "hi", "backend": "supabase", "operation": "contains", "question": "specimens से वे पंक्तियाँ दिखाएं जिनमें display_name Pune के समान है, मामला अंतर नजरअंदाज करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE LOWER(display_name) LIKE LOWER('%Pune%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(display_name) LIKE LOWER('%Pune%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2ca6667f987742cde1e7_en", "language": "en", "backend": "mysql", "operation": "write", "question": "Expunge every row from specimens.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "3fdbfd3a4d0df901f143_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gte", "question": "output records from specimens where amount is not below 200", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE amount >= 200;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE amount >= 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e114d1f216088c4302b2_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "distinct", "question": "i need the distinct department values from specimens", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT DISTINCT department FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT department FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e6380b8531b4b3eb7e38_en", "language": "en", "backend": "mysql", "operation": "project", "question": "Display name from specimens as the sole result.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "253e89e4cdbbd5c2d0f9_en", "language": "en", "backend": "mysql", "operation": "group_count", "question": "Find the count of rows per status in specimens.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT status, COUNT(*) FROM specimens GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT status, COUNT(*) FROM specimens GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "253e89e4cdbbd5c2d0f9_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_count", "question": "give me the count for each status rows in specimens", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT status, COUNT(*) FROM specimens GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT status, COUNT(*) FROM specimens GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0abe13c841fd7c3ac885_hi", "language": "hi", "backend": "supabase", "operation": "having", "question": "कृपया inspections में 5000 से अधिक पंक्तियों वाले region समूहों और उनकी गणना को प्रदर्शित करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT region, COUNT(*) FROM inspections GROUP BY region HAVING COUNT(*) > 5000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM inspections GROUP BY region HAVING COUNT(*) > 5000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e7d754ed36b247db55d3_en", "language": "en", "backend": "mysql", "operation": "not_null", "question": "Present the data from specimens where description is not null.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e7d754ed36b247db55d3_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "not_null", "question": "Find the rows in specimens where description is not null.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5a8f2e207d0efe5b018c_hi", "language": "hi", "backend": "supabase", "operation": "list_tables", "question": "इस डेटाबेस की तालिकाओं का विवरण दीजिए।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "49c2e66e48d062e5c69c_hinglish", "language": "hinglish", "backend": "mysql", "operation": "write", "question": "specimens ka pura content hatwa do.", "expected": {"action": "clarify", "question": "Yeh connection read-only hai. Kya SELECT query chahiye?"}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}}
{"id": "d30264bbc3b08aedbc51_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sum", "question": "i want the total score of exhibits", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT SUM(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "aa4f0e1f29573e237ee1_hi", "language": "hi", "backend": "supabase", "operation": "list_tables", "question": "इस डेटाबेस की तालिकाओं का विवरण दीजिए।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "4fb8670235a435c82c88_hi", "language": "hi", "backend": "supabase", "operation": "describe", "question": "टेबल exhibits के कॉलम संरचना और टाइप्स दिखाइए।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_exhibits", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "8ebc9e868ee558a8ddfe_hi", "language": "hi", "backend": "supabase", "operation": "ambiguous", "question": "specimens में उपलब्ध सर्वोत्तम रिकॉर्ड प्रिंट करें।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "2ca6667f987742cde1e7_hinglish", "language": "hinglish", "backend": "mysql", "operation": "write", "question": "specimens ka pura content hatwa do.", "expected": {"action": "clarify", "question": "Yeh connection read-only hai. Kya SELECT query chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"Yeh connection read-only hai. Kya SELECT query chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "aa867c871299c26d5402_en", "language": "en", "backend": "mysql", "operation": "lt", "question": "Query specimens for rows where rating is less than 1000.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE rating < 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE rating < 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4c5f91f7793bd0200f31_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "top", "question": "I need 1 rows from specimens with amount sorted descending.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens ORDER BY amount DESC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY amount DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "eab95ae98810f2228ba6_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "year", "question": "Fetch specimens entries where start_date year is 2026.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens WHERE YEAR(start_date) = 2026;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE YEAR(start_date) = 2026;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d0346c74f1cfe556b2cb_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project_eq", "question": "give me description from inspections where status is Books", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT description FROM inspections WHERE status = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT description FROM inspections WHERE status = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1523073b9b419fc60ded_en", "language": "en", "backend": "mysql", "operation": "count_eq", "question": "Provide the row count for specimens where city is Books.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT COUNT(*) FROM specimens WHERE city = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens WHERE city = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1523073b9b419fc60ded_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count_eq", "question": "Please count the rows in specimens where city is Books.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT COUNT(*) FROM specimens WHERE city = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens WHERE city = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "16fbb58e3a07a1240f50_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "avg", "question": "determine the mean score in exhibits", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT AVG(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT AVG(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e71814820a36875f0ee5_en", "language": "en", "backend": "mysql", "operation": "not_null", "question": "Present the data from specimens where description is not null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e71814820a36875f0ee5_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "not_null", "question": "Find the rows in specimens where description is not null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5ce4cc7ea9d93a0aa4c5_hi", "language": "hi", "backend": "supabase", "operation": "sum", "question": "reservations में quantity के सभी मानों का योग करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT SUM(quantity) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT SUM(quantity) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "adff4eabe2639c0c128c_hinglish", "language": "hinglish", "backend": "mysql", "operation": "all", "question": "inspections ke saare records get karo.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "cd5bb3382b9e6c8604e8_hinglish", "language": "hinglish", "backend": "mysql", "operation": "max", "question": "specimens mein rating ka sabse bada number kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(rating) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(rating) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7094d70d08791284b632_hi", "language": "hi", "backend": "supabase", "operation": "min", "question": "reservations में salary का न्यूनतम परिमाण क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT MIN(salary) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT MIN(salary) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1d82ef2b9527e797c466_hinglish", "language": "hinglish", "backend": "supabase", "operation": "date_after", "question": "inspections mein se wo rows chuno jinke order_date mein 2022-04-15 ya uske baad ki date hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE order_date >= '2022-04-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE order_date >= '2022-04-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "26db633822b8319f83ec_hi", "language": "hi", "backend": "supabase", "operation": "contains", "question": "reservations से वे पंक्तियाँ दिखाएं जिनमें name active के समान है, मामला अंतर नजरअंदाज करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE LOWER(name) LIKE LOWER('%active%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE LOWER(name) LIKE LOWER('%active%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "84b16996198346dc9882_hi", "language": "hi", "backend": "mysql", "operation": "missing_schema", "question": "कृपया exhibits के लिए SQL लिखने से पहले उपलब्ध तालिकाओं का पता लगाएं, क्योंकि मैंने डेटाबेस स्कीमा नहीं दी है।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "81b12886ddb859bb71db_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count_eq", "question": "how many inspections rows where city is pending", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM inspections WHERE city = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM inspections WHERE city = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9aed1992a793fbf97601_hinglish", "language": "hinglish", "backend": "mysql", "operation": "min", "question": "exhibits mein price ka minimum reading kya hai?", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT MIN(price) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT MIN(price) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8a3194b9f492b50b8100_en", "language": "en", "backend": "mysql", "operation": "distinct", "question": "I want to see the distinct category values available in specimens.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT category FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8a3194b9f492b50b8100_hinglish", "language": "hinglish", "backend": "mysql", "operation": "distinct", "question": "Mujhe specimens mein available category ke distinct values dekhne hain.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT category FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2164b376f732a1c371ab_en", "language": "en", "backend": "mysql", "operation": "group_count", "question": "Find the count of rows per kind in specimens.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM specimens GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2164b376f732a1c371ab_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_count", "question": "give me the count for each kind rows in specimens", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM specimens GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ed474bd8a459cbfaa5d2_hi", "language": "hi", "backend": "supabase", "operation": "year", "question": "specimens में recorded_at में 2021 वाले रिकॉर्ड दिखाएं।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE EXTRACT(YEAR FROM recorded_at) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c0b9cf3b44d96839f256_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "min", "question": "What is the floor balance in inspections?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(balance) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a29fdee3339da6711080_en", "language": "en", "backend": "mysql", "operation": "bottom", "question": "Present the 20 minimal rating entries from specimens in ascending order.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens ORDER BY rating ASC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY rating ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a29fdee3339da6711080_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "bottom", "question": "print the 20 minimal rating rows in specimens from lowest up", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens ORDER BY rating ASC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY rating ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "61b3db48f1ff12dc69e1_hi", "language": "hi", "backend": "supabase", "operation": "group_sum", "question": "exhibits में kind के अनुसार quantity का कुल कृपया दें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT kind, SUM(quantity) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT kind, SUM(quantity) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6d9e81a9f52b8d33060a_hinglish", "language": "hinglish", "backend": "mysql", "operation": "all", "question": "exhibits ke saare records get karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bc8593644a72fe8b9c78_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "min", "question": "What is the floor score in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "69ac9800bd7a3ee23ccc_en", "language": "en", "backend": "mysql", "operation": "distinct", "question": "I want to see the distinct category values available in specimens.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT category FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "69ac9800bd7a3ee23ccc_hinglish", "language": "hinglish", "backend": "mysql", "operation": "distinct", "question": "Mujhe specimens mein available category ke distinct values dekhne hain.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT category FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f1f8414c2d47f5396d69_hinglish", "language": "hinglish", "backend": "supabase", "operation": "date_after", "question": "exhibits mein se wo rows chuno jinke created_at mein 2023-03-15 ya uske baad ki date hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE created_at >= '2023-03-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE created_at >= '2023-03-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3fdbfd3a4d0df901f143_en", "language": "en", "backend": "mysql", "operation": "gte", "question": "Show the data from specimens where amount is 200 or above.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE amount >= 200;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE amount >= 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bfb3f8261ddb2fc2bf28_en", "language": "en", "backend": "mysql", "operation": "bottom", "question": "Present the 3 minimal price entries from specimens in ascending order.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens ORDER BY price ASC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens ORDER BY price ASC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bfb3f8261ddb2fc2bf28_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "bottom", "question": "print the 3 minimal price rows in specimens from lowest up", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens ORDER BY price ASC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens ORDER BY price ASC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "eab95ae98810f2228ba6_en", "language": "en", "backend": "mysql", "operation": "year", "question": "Show me the specimens rows for 2026 using start_date.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens WHERE YEAR(start_date) = 2026;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE YEAR(start_date) = 2026;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "24661fe309a7c9930145_en", "language": "en", "backend": "mysql", "operation": "month", "question": "Select entries from specimens where recorded_at corresponds to 1.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE MONTH(recorded_at) = 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE MONTH(recorded_at) = 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d0346c74f1cfe556b2cb_en", "language": "en", "backend": "mysql", "operation": "project_eq", "question": "List the description from inspections where status is Books.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT description FROM inspections WHERE status = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT description FROM inspections WHERE status = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d09477cb849c0af3827d_en", "language": "en", "backend": "mysql", "operation": "ambiguous", "question": "Print the best available records in exhibits.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "d09477cb849c0af3827d_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "ambiguous", "question": "Give me the top records in exhibits.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "78fd6e66c9d726084543_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count", "question": "how many lines in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT COUNT(*) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "aeeb76897a32b9fd2df4_hi", "language": "hi", "backend": "supabase", "operation": "min", "question": "reservations में amount का न्यूनतम परिमाण क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT MIN(amount) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT MIN(amount) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "41ea5651dda516a4ac30_en", "language": "en", "backend": "supabase", "operation": "date_after", "question": "Extract rows from reservations where order_date falls on 2023-12-15 or beyond.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE order_date >= '2023-12-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE order_date >= '2023-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e7d754ed36b247db55d3_hinglish", "language": "hinglish", "backend": "mysql", "operation": "not_null", "question": "specimens ka data present karo jahan description null nahi hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b9c176c07809f42564a0_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "top", "question": "I need 5 rows from inspections with amount sorted descending.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2162ee1266aed9320937_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "write", "question": "Remove all records from inspections now.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "23c0aa3a4596f995ad2a_en", "language": "en", "backend": "mysql", "operation": "count", "question": "What is the volume of data rows in specimens?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9aed1992a793fbf97601_en", "language": "en", "backend": "mysql", "operation": "min", "question": "What is the baseline price value in exhibits?", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT MIN(price) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT MAX(price) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "5d97abdb52189efbe117_en", "language": "en", "backend": "mysql", "operation": "group_sum", "question": "Please provide the total amount for each status in exhibits.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT status, SUM(amount) FROM exhibits GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT status, SUM(amount) FROM exhibits GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "58eb6ab2fb9c69becdbd_hinglish", "language": "hinglish", "backend": "supabase", "operation": "date_after", "question": "reservations mein se wo rows chuno jinke created_at mein 2022-08-15 ya uske baad ki date hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE created_at >= '2022-08-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE created_at >= '2022-08-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d30264bbc3b08aedbc51_en", "language": "en", "backend": "mysql", "operation": "sum", "question": "Output the total score for the exhibits entity.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT SUM(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5cb3a450bcdc957b7ba5_hi", "language": "hi", "backend": "mysql", "operation": "missing_schema", "question": "कृपया specimens के लिए SQL लिखने से पहले उपलब्ध तालिकाओं का पता लगाएं, क्योंकि मैंने डेटाबेस स्कीमा नहीं दी है।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "55cd623d82474692b1a9_en", "language": "en", "backend": "mysql", "operation": "count", "question": "What is the volume of data rows in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e2b48c4ec5252934cb20_en", "language": "en", "backend": "mysql", "operation": "count_eq", "question": "Provide the row count for inspections where city is pending.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT COUNT(*) FROM inspections WHERE city = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM inspections WHERE city = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e2b48c4ec5252934cb20_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count_eq", "question": "Please count the rows in inspections where city is pending.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT COUNT(*) FROM inspections WHERE city = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM inspections WHERE city = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "eab95ae98810f2228ba6_hinglish", "language": "hinglish", "backend": "mysql", "operation": "year", "question": "specimens mein start_date mein 2026 wale records dikhao.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens WHERE YEAR(start_date) = 2026;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE YEAR(start_date) = 2026;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "24661fe309a7c9930145_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "month", "question": "extract rows from specimens where recorded_at month is 1", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE MONTH(recorded_at) = 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE MONTH(recorded_at) = 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "16fbb58e3a07a1240f50_en", "language": "en", "backend": "mysql", "operation": "avg", "question": "Return the average score in exhibits.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT AVG(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT AVG(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "16fbb58e3a07a1240f50_hinglish", "language": "hinglish", "backend": "mysql", "operation": "avg", "question": "exhibits me score ka average return karo.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT AVG(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT AVG(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "feb75e02c031a950e223_en", "language": "en", "backend": "mysql", "operation": "group_sum", "question": "Please provide the total cost for each category in inspections.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT category, SUM(cost) FROM inspections GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT category, SUM(cost) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "175564da35a6c139e5d8_hi", "language": "hi", "backend": "supabase", "operation": "contains", "question": "inspections से वे पंक्तियाँ दिखाएं जिनमें display_name North के समान है, मामला अंतर नजरअंदाज करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE LOWER(display_name) LIKE LOWER('%North%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE LOWER(display_name) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d09477cb849c0af3827d_hinglish", "language": "hinglish", "backend": "mysql", "operation": "ambiguous", "question": "exhibits mein available best records print karo.", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "e71814820a36875f0ee5_hinglish", "language": "hinglish", "backend": "mysql", "operation": "not_null", "question": "specimens ka data present karo jahan description null nahi hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "78fd6e66c9d726084543_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count", "question": "Batao exhibits mein kitni entries hain.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT COUNT(*) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a4cdc86496ddd0c1cc90_hi", "language": "hi", "backend": "mysql", "operation": "avg", "question": "specimens में cost का औसत लौटाएँ।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT AVG(cost) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT AVG(cost) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "253e89e4cdbbd5c2d0f9_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_count", "question": "specimens me pratyek status ke liye rows ki sankhya dhundo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT status, COUNT(*) FROM specimens GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT status, COUNT(*) FROM specimens GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1d7ab92421915fd47ebe_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "year", "question": "Fetch inspections entries where created_at year is 2024.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections WHERE YEAR(created_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE YEAR(created_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "cab55a7429427fbc5093_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "distinct", "question": "i need the distinct department values from inspections", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT department FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7506912696ddf94cfae4_en", "language": "en", "backend": "mysql", "operation": "max", "question": "What is the largest number for amount in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(amount) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(amount) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7506912696ddf94cfae4_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "max", "question": "what is the upper bound of amount in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(amount) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(amount) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5d97abdb52189efbe117_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_sum", "question": "I need to see the total amount for each status in exhibits", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT status, SUM(amount) FROM exhibits GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT status, SUM(amount) FROM exhibits GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e8f90532d34c12209a83_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gte", "question": "output records from inspections where salary is not below 1500", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE salary >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c0b9cf3b44d96839f256_hinglish", "language": "hinglish", "backend": "mysql", "operation": "min", "question": "inspections mein balance ka minimum reading kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(balance) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b45bf4e81303e2b4da0f_en", "language": "en", "backend": "mysql", "operation": "gt", "question": "Bring up the rows in specimens where price is over 1.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens WHERE price > 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE price > 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8d2022223833a85bd2d1_en", "language": "en", "backend": "mysql", "operation": "max", "question": "What is the largest number for salary in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT MAX(salary) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT MAX(salary) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8d2022223833a85bd2d1_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "max", "question": "what is the upper bound of salary in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT MAX(salary) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT MAX(salary) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bc8593644a72fe8b9c78_hinglish", "language": "hinglish", "backend": "mysql", "operation": "min", "question": "exhibits mein score ka minimum reading kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "aa867c871299c26d5402_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lt", "question": "can you select the rows from specimens where rating is less than 1000?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE rating < 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE rating < 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3fdbfd3a4d0df901f143_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gte", "question": "specimens se rows dikhao jahan amount 200 ya usse bada hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE amount >= 200;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE amount >= 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e114d1f216088c4302b2_en", "language": "en", "backend": "mysql", "operation": "distinct", "question": "I want to see the distinct department values available in specimens.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT DISTINCT department FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT department FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e114d1f216088c4302b2_hinglish", "language": "hinglish", "backend": "mysql", "operation": "distinct", "question": "Mujhe specimens mein available department ke distinct values dekhne hain.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT DISTINCT department FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT department FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4c5f91f7793bd0200f31_hinglish", "language": "hinglish", "backend": "mysql", "operation": "top", "question": "Top 1 rows specimens se lo, amount ke hisaab se highest first.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens ORDER BY amount DESC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY amount DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d0346c74f1cfe556b2cb_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project_eq", "question": "inspections se description ki list do jahan status Books hai.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT description FROM inspections WHERE status = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT description FROM inspections WHERE status = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6519cd640e9e4ab22a6e_en", "language": "en", "backend": "mysql", "operation": "gt", "question": "Bring up the rows in specimens where score is over 20.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE score > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE score > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "feb75e02c031a950e223_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_sum", "question": "I need to see the total cost for each category in inspections", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT category, SUM(cost) FROM inspections GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT category, SUM(cost) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "860c064ccde26d474c46_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project", "question": "bas exhibits ka display_name do please", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a6079b58d4365a8f70c8_hinglish", "language": "hinglish", "backend": "mysql", "operation": "null", "question": "inspections se title null wali rows select karo.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "508d2834a4743665db1f_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "distinct", "question": "i need the distinct kind values from exhibits", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT DISTINCT kind FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT DISTINCT kind FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1d7ab92421915fd47ebe_en", "language": "en", "backend": "mysql", "operation": "year", "question": "Show me the inspections rows for 2024 using created_at.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections WHERE YEAR(created_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE YEAR(created_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1d7ab92421915fd47ebe_hinglish", "language": "hinglish", "backend": "mysql", "operation": "year", "question": "inspections mein created_at mein 2024 wale records dikhao.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections WHERE YEAR(created_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE YEAR(created_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "35406fa024e66bcb5dd7_hi", "language": "hi", "backend": "supabase", "operation": "lt", "question": "specimens से उन पंक्तियों का क्वेरी करें जहाँ score 10000 से कम है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE score < 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE score < 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8efcf517aee64807a8e5_hi", "language": "hi", "backend": "mysql", "operation": "avg", "question": "specimens में balance का औसत लौटाएँ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6972a0b38fd4a8758723_en", "language": "en", "backend": "mysql", "operation": "group_sum", "question": "Please provide the total score for each kind in exhibits.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT kind, SUM(score) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT kind, SUM(score) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d8f9a887c52e766a8e14_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "year", "question": "Fetch exhibits entries where start_date year is 2023.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE YEAR(start_date) = 2023;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE YEAR(start_date) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "81b12886ddb859bb71db_en", "language": "en", "backend": "mysql", "operation": "count_eq", "question": "Provide the row count for inspections where city is pending.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM inspections WHERE city = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM inspections WHERE city = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "81b12886ddb859bb71db_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count_eq", "question": "Please count the rows in inspections where city is pending.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM inspections WHERE city = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM inspections WHERE city = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7506912696ddf94cfae4_hinglish", "language": "hinglish", "backend": "mysql", "operation": "max", "question": "exhibits mein amount ka sabse bada number kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(amount) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(amount) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5890e52f983d16adbae7_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "top", "question": "I need 3 rows from inspections with quantity sorted descending.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY quantity DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY quantity DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2164b376f732a1c371ab_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_count", "question": "specimens me pratyek kind ke liye rows ki sankhya dhundo.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM specimens GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5d97abdb52189efbe117_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_sum", "question": "exhibits me status ke hisaab se amount ka total do please.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT status, SUM(amount) FROM exhibits GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT status, SUM(amount) FROM exhibits GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "15b265bd8bbdb3515605_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project_eq", "question": "give me display_name from inspections where kind is Electronics", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT display_name FROM inspections WHERE kind = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT display_name FROM inspections WHERE kind = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d5fc4318c9aedb495460_en", "language": "en", "backend": "mysql", "operation": "lt", "question": "Query inspections for rows where price is less than 50.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE price < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE price < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0352841d74f7bd7955ea_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lte", "question": "show the inspections table rows where score is at most 3", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections WHERE score <= 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE score <= 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d61a27de98bb7d44f15d_en", "language": "en", "backend": "mysql", "operation": "count", "question": "What is the volume of data rows in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens WHERE department = 'volume';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "d30264bbc3b08aedbc51_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sum", "question": "exhibits entity ke liye score ka total output do.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT SUM(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "64372a29664a7eaddbe2_hi", "language": "hi", "backend": "mysql", "operation": "avg", "question": "specimens में amount का औसत लौटाएँ।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT AVG(amount) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT AVG(amount) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c0b9cf3b44d96839f256_en", "language": "en", "backend": "mysql", "operation": "min", "question": "What is the baseline balance value in inspections?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(balance) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "cfb6d9bde68e2cdc5bf9_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project_eq", "question": "give me name from reservations where category is Books", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT name FROM reservations WHERE category = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT name FROM reservations WHERE category = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b45bf4e81303e2b4da0f_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gt", "question": "show me the entries in specimens where price is strictly greater than 1", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens WHERE price > 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE price > 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "864b7a7bf739071bb030_hi", "language": "hi", "backend": "supabase", "operation": "and", "question": "specimens में वे पंक्तियाँ खोजें जहाँ status Pune है और balance 0 से ऊपर है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE status = 'Pune' AND balance > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE status = 'Pune' AND balance > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8d2022223833a85bd2d1_hinglish", "language": "hinglish", "backend": "mysql", "operation": "max", "question": "exhibits mein salary ka sabse bada number kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT MAX(salary) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT MAX(salary) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bc8593644a72fe8b9c78_en", "language": "en", "backend": "mysql", "operation": "min", "question": "What is the baseline score value in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d96328fdb5748b255271_en", "language": "en", "backend": "mysql", "operation": "group_sum", "question": "Please provide the total salary for each category in inspections.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT category, SUM(salary) FROM inspections GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT category, SUM(salary) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "24661fe309a7c9930145_hinglish", "language": "hinglish", "backend": "mysql", "operation": "month", "question": "Show me the data from specimens where recorded_at ka month 1 hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE MONTH(recorded_at) = 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE MONTH(recorded_at) = 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "29bdf95ec36d6924a576_hi", "language": "hi", "backend": "supabase", "operation": "contains", "question": "exhibits से वे पंक्तियाँ दिखाएं जिनमें item_name O'Reilly के समान है, मामला अंतर नजरअंदाज करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE LOWER(item_name) LIKE LOWER('%O''Reilly%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE LOWER(item_name) LIKE LOWER('%O''Reilly%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e9cfe7f6c91a2434d315_hi", "language": "hi", "backend": "supabase", "operation": "final", "question": "डेटाबेस टूल ने 102 पंक्तियां वापस कीं। कितनी पंक्तियां लौटीं?", "expected": {"action": "answer", "text": "टूल ने 102 पंक्तियाँ लौटाईं।"}, "output": "{\"action\":\"answer\",\"text\":\"टूल ने 102 पंक्तियाँ लौटाईं।\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "e6380b8531b4b3eb7e38_hi", "language": "hi", "backend": "mysql", "operation": "project", "question": "सिर्फ specimens का name दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "19d562dac8a4125562f6_en", "language": "en", "backend": "mysql", "operation": "eq", "question": "Bring up all rows from inspections where city has the value Delhi.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM inspections WHERE city = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE city = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "19d562dac8a4125562f6_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "eq", "question": "Get the full list of rows in inspections where city is Delhi.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM inspections WHERE city = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE city = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6519cd640e9e4ab22a6e_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gt", "question": "show me the entries in specimens where score is strictly greater than 20", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE score > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE score > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "28cb6c4bf0d7caac91c9_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "or", "question": "Display rows from exhibits where kind is Mumbai or Electronics.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'Electronics');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8439e708b96823b39172_en", "language": "en", "backend": "mysql", "operation": "sort_asc", "question": "View all records in inspections sorted by amount from least to greatest.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "feb75e02c031a950e223_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_sum", "question": "inspections me category ke hisaab se cost ka total do please.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT category, SUM(cost) FROM inspections GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT category, SUM(cost) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a76838f987782ef2d2ed_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "describe", "question": "Look up the columns and types for inspections.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "inspections"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "860c064ccde26d474c46_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project", "question": "display display_name from exhibits and nothing else", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9eee465dd6b4c3a090c6_en", "language": "en", "backend": "mysql", "operation": "eq", "question": "Bring up all rows from specimens where region has the value Pune.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE region = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE region = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9eee465dd6b4c3a090c6_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "eq", "question": "Get the full list of rows in specimens where region is Pune.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE region = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE region = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a6079b58d4365a8f70c8_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "null", "question": "Show me the rows from inspections where title is null.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "41ea5651dda516a4ac30_hinglish", "language": "hinglish", "backend": "supabase", "operation": "date_after", "question": "reservations mein se wo rows chuno jinke order_date mein 2023-12-15 ya uske baad ki date hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE order_date >= '2023-12-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE order_date >= '2023-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b9c176c07809f42564a0_hinglish", "language": "hinglish", "backend": "mysql", "operation": "top", "question": "Top 5 rows inspections se lo, amount ke hisaab se highest first.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6972a0b38fd4a8758723_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_sum", "question": "I need to see the total score for each kind in exhibits", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT kind, SUM(score) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT kind, SUM(score) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d8f9a887c52e766a8e14_en", "language": "en", "backend": "mysql", "operation": "year", "question": "Show me the exhibits rows for 2023 using start_date.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE YEAR(start_date) = 2023;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE YEAR(start_date) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d8f9a887c52e766a8e14_hinglish", "language": "hinglish", "backend": "mysql", "operation": "year", "question": "exhibits mein start_date mein 2023 wale records dikhao.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE YEAR(start_date) = 2023;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE YEAR(start_date) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2162ee1266aed9320937_en", "language": "en", "backend": "mysql", "operation": "write", "question": "Expunge every row from inspections.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE price > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}}
{"id": "2162ee1266aed9320937_hinglish", "language": "hinglish", "backend": "mysql", "operation": "write", "question": "inspections ka pura content hatwa do.", "expected": {"action": "clarify", "question": "Yeh connection read-only hai. Kya SELECT query chahiye?"}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}}
{"id": "c023a5b6f95173ae856d_en", "language": "en", "backend": "mysql", "operation": "gt", "question": "Bring up the rows in exhibits where score is over 50.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE score > 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bd760f1ca7aff546e852_en", "language": "en", "backend": "mysql", "operation": "bottom", "question": "Present the 5 minimal balance entries from exhibits in ascending order.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits ORDER BY balance ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY balance ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bd760f1ca7aff546e852_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "bottom", "question": "print the 5 minimal balance rows in exhibits from lowest up", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits ORDER BY balance ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY balance ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "00c24a7cc97ee9debf36_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "contains", "question": "List exhibits rows where title equals pending, ignore case differences.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM exhibits WHERE LOWER(title) LIKE LOWER('%pending%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE LOWER(title) LIKE LOWER('%pending%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "15b265bd8bbdb3515605_en", "language": "en", "backend": "mysql", "operation": "project_eq", "question": "List the display_name from inspections where kind is Electronics.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT display_name FROM inspections WHERE kind = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT display_name FROM inspections WHERE kind = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e8f90532d34c12209a83_en", "language": "en", "backend": "mysql", "operation": "gte", "question": "Show the data from inspections where salary is 1500 or above.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE salary >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0352841d74f7bd7955ea_en", "language": "en", "backend": "mysql", "operation": "lte", "question": "Query inspections for rows where score is not greater than 3.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections WHERE score <= 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE score <= 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6f28c044459ce99cf405_en", "language": "en", "backend": "mysql", "operation": "max", "question": "What is the largest number for quantity in exhibits?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT MAX(quantity) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT MAX(quantity) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6f28c044459ce99cf405_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "max", "question": "what is the upper bound of quantity in exhibits?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT MAX(quantity) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT MAX(quantity) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "cfb6d9bde68e2cdc5bf9_en", "language": "en", "backend": "mysql", "operation": "project_eq", "question": "List the name from reservations where category is Books.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT name FROM reservations WHERE category = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT name FROM reservations WHERE category = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4e4b83d8af9eb4c5efc8_en", "language": "en", "backend": "mysql", "operation": "bottom", "question": "Present the 5 minimal amount entries from inspections in ascending order.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4e4b83d8af9eb4c5efc8_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "bottom", "question": "print the 5 minimal amount rows in inspections from lowest up", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d96328fdb5748b255271_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_sum", "question": "I need to see the total salary for each category in inspections", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT category, SUM(salary) FROM inspections GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT category, SUM(salary) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2f545db764e4c6cd353d_en", "language": "en", "backend": "mysql", "operation": "gt", "question": "Bring up the rows in exhibits where salary is over 0.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE salary > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE salary > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "aa867c871299c26d5402_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lt", "question": "specimens se un rows ko query karo jahan rating 1000 se kam hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE rating < 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE rating < 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c466f263014e2eb31cf5_en", "language": "en", "backend": "mysql", "operation": "max", "question": "What is the largest number for rating in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT MAX(rating) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT MAX(rating) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c466f263014e2eb31cf5_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "max", "question": "what is the upper bound of rating in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT MAX(rating) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT MAX(rating) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4c5f91f7793bd0200f31_en", "language": "en", "backend": "mysql", "operation": "top", "question": "Give me the top 1 rows of specimens sorted by amount from high to low.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens ORDER BY amount DESC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY amount DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "19d562dac8a4125562f6_hinglish", "language": "hinglish", "backend": "mysql", "operation": "eq", "question": "inspections se wo saari rows lao jahan city ka value Delhi hai.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM inspections WHERE city = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE city = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "28cb6c4bf0d7caac91c9_hinglish", "language": "hinglish", "backend": "mysql", "operation": "or", "question": "exhibits mein kind Mumbai ya Electronics wali rows dikhao na.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'Electronics');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a7efbaf306821c7ab040_hi", "language": "hi", "backend": "supabase", "operation": "year", "question": "reservations में order_date में 2023 वाले रिकॉर्ड दिखाएं।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE EXTRACT(YEAR FROM order_date) = 2023;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM order_date) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9f8fa242b083f38bc5f8_en", "language": "en", "backend": "mysql", "operation": "month", "question": "Select entries from inspections where order_date corresponds to 4.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE MONTH(order_date) = 4;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE MONTH(order_date) = 4;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2c31add7d7b45843dec1_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project_eq", "question": "give me display_name from inspections where kind is Pune", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM inspections WHERE kind = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b02f421828127e2eb96b_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lte", "question": "show the exhibits table rows where price is at most 2", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price <= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price <= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6972a0b38fd4a8758723_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_sum", "question": "exhibits me kind ke hisaab se score ka total do please.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT kind, SUM(score) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT kind, SUM(score) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e62cfd2a6789616a503f_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "describe", "question": "Look up the columns and types for reservations.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "reservations"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "c023a5b6f95173ae856d_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gt", "question": "show me the entries in exhibits where score is strictly greater than 50", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE score > 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e8f90532d34c12209a83_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gte", "question": "inspections se rows dikhao jahan salary 1500 ya usse bada hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE salary >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6f28c044459ce99cf405_hinglish", "language": "hinglish", "backend": "mysql", "operation": "max", "question": "exhibits mein quantity ka sabse bada number kya hai?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT MAX(quantity) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT MAX(quantity) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5aaaf945b8fba37eadf4_en", "language": "en", "backend": "mysql", "operation": "sort_asc", "question": "View all records in inspections sorted by price from least to greatest.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY price ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "85e339da2fb28bae5395_hi", "language": "hi", "backend": "supabase", "operation": "date_after", "question": "reservations में से वे पंक्तियाँ चुनें जिनके order_date में 2024-10-15 या उससे बाद की तारीख है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE order_date >= '2024-10-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE order_date >= '2024-10-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "cfb6d9bde68e2cdc5bf9_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project_eq", "question": "reservations se name ki list do jahan category Books hai.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT name FROM reservations WHERE category = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT name FROM reservations WHERE category = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d96328fdb5748b255271_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_sum", "question": "inspections me category ke hisaab se salary ka total do please.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT category, SUM(salary) FROM inspections GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT category, SUM(salary) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2ed720de1d1005daa102_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "describe", "question": "Look up the columns and types for reservations.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "reservations"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "2f545db764e4c6cd353d_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gt", "question": "show me the entries in exhibits where salary is strictly greater than 0", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE salary > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE salary > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c466f263014e2eb31cf5_hinglish", "language": "hinglish", "backend": "mysql", "operation": "max", "question": "exhibits mein rating ka sabse bada number kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT MAX(rating) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT MAX(rating) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "46b2f9dcb5c441ef3e8e_en", "language": "en", "backend": "mysql", "operation": "group_sum", "question": "Please provide the total rating for each kind in exhibits.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT kind, SUM(rating) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT kind, SUM(rating) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5c6734bdfead38b51752_hi", "language": "hi", "backend": "supabase", "operation": "gte", "question": "specimens से वे पंक्तियाँ दिखाएँ जहाँ quantity 10000 या उससे बड़ा है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE quantity >= 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE quantity >= 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "28cb6c4bf0d7caac91c9_en", "language": "en", "backend": "mysql", "operation": "or", "question": "Bring up rows from exhibits where kind is either Mumbai or Electronics.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'Electronics');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8439e708b96823b39172_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_asc", "question": "can i see all rows from inspections sorted by amount from small to large", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8439e708b96823b39172_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_asc", "question": "inspections se saara data amount ke badhte hue order mein lo aur dikhao.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a76838f987782ef2d2ed_en", "language": "en", "backend": "mysql", "operation": "describe", "question": "Reveal the column structure and types for inspections.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "inspections"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "9eee465dd6b4c3a090c6_hinglish", "language": "hinglish", "backend": "mysql", "operation": "eq", "question": "specimens se wo saari rows lao jahan region ka value Pune hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE region = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE region = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7e281ac4ee7d2d835606_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lte", "question": "show the inspections table rows where rating is at most 1", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE rating <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE rating <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a6079b58d4365a8f70c8_en", "language": "en", "backend": "mysql", "operation": "null", "question": "I want to view the rows in inspections where title is null.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7fb4f6e9f1b68e05cfd9_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_desc", "question": "retrieve all records from reservations sorted by salary high to low", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM reservations ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "02981b7ccddb9b467260_en", "language": "en", "backend": "mysql", "operation": "sort_asc", "question": "View all records in exhibits sorted by quantity from least to greatest.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits ORDER BY quantity ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY quantity ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9f8fa242b083f38bc5f8_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "month", "question": "extract rows from inspections where order_date month is 4", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE MONTH(order_date) = 4;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE MONTH(order_date) = 4;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2c31add7d7b45843dec1_en", "language": "en", "backend": "mysql", "operation": "project_eq", "question": "List the display_name from inspections where kind is Pune.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM inspections WHERE kind = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b02f421828127e2eb96b_en", "language": "en", "backend": "mysql", "operation": "lte", "question": "Query exhibits for rows where price is not greater than 2.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price <= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price <= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "cab55a7429427fbc5093_en", "language": "en", "backend": "mysql", "operation": "distinct", "question": "I want to see the distinct department values available in inspections.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT department FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "cab55a7429427fbc5093_hinglish", "language": "hinglish", "backend": "mysql", "operation": "distinct", "question": "Mujhe inspections mein available department ke distinct values dekhne hain.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT department FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b9c176c07809f42564a0_en", "language": "en", "backend": "mysql", "operation": "top", "question": "Give me the top 5 rows of inspections sorted by amount from high to low.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a060620c638d2dc3759e_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project", "question": "bas reservations ka item_name do please", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5890e52f983d16adbae7_hinglish", "language": "hinglish", "backend": "mysql", "operation": "top", "question": "Top 3 rows inspections se lo, quantity ke hisaab se highest first.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY quantity DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY quantity DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "35b25e2b46b13ad30caf_en", "language": "en", "backend": "mysql", "operation": "ambiguous", "question": "Print the best available records in reservations.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "35b25e2b46b13ad30caf_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "ambiguous", "question": "Give me the top records in reservations.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "35b25e2b46b13ad30caf_hinglish", "language": "hinglish", "backend": "mysql", "operation": "ambiguous", "question": "reservations mein available best records print karo.", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "15b265bd8bbdb3515605_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project_eq", "question": "inspections se display_name ki list do jahan kind Electronics hai.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT display_name FROM inspections WHERE kind = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT display_name FROM inspections WHERE kind = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d5fc4318c9aedb495460_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lt", "question": "can you select the rows from inspections where price is less than 50?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE price < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE price < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0352841d74f7bd7955ea_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lte", "question": "inspections se wo rows dhundo jaha score 3 se bada nahi hai.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections WHERE score <= 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE score <= 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2e9e0c695ac50e3982d7_en", "language": "en", "backend": "mysql", "operation": "month", "question": "Select entries from exhibits where order_date corresponds to 12.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE MONTH(order_date) = 12;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE MONTH(order_date) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b45bf4e81303e2b4da0f_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gt", "question": "specimens se un rows ko display karo jahan price 1 se more hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens WHERE price > 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE price > 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "208e9ccf17a923a46aac_hinglish", "language": "hinglish", "backend": "mysql", "operation": "null", "question": "inspections se item_name null wali rows select karo.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "46b2f9dcb5c441ef3e8e_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_sum", "question": "I need to see the total rating for each kind in exhibits", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT kind, SUM(rating) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT kind, SUM(rating) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6519cd640e9e4ab22a6e_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gt", "question": "specimens se un rows ko display karo jahan score 20 se more hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE score > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE score > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "393d51bc9e30d68b474e_hi", "language": "hi", "backend": "supabase", "operation": "and", "question": "inspections में वे पंक्तियाँ खोजें जहाँ status active है और cost 5 से ऊपर है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE status = 'active' AND cost > 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE status = 'active' AND cost > 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a0bc7565d9d93d546770_hinglish", "language": "hinglish", "backend": "mysql", "operation": "null", "question": "inspections se item_name null wali rows select karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "16fbb58e3a07a1240f50_hi", "language": "hi", "backend": "mysql", "operation": "avg", "question": "exhibits में score का औसत लौटाएँ।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT AVG(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT AVG(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b0f56c14239ba5d0227f_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "distinct", "question": "i need the distinct region values from reservations", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT DISTINCT region FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT region FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a76838f987782ef2d2ed_hinglish", "language": "hinglish", "backend": "mysql", "operation": "describe", "question": "Mujhe inspections ke columns aur types ke baare mein info do.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "inspections"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "5f07a4c7c387316664f1_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "missing_value", "question": "execute filter on exhibits for department, but i have not specified a value", "expected": {"action": "clarify", "question": "Which value should department equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should department equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "860c064ccde26d474c46_en", "language": "en", "backend": "mysql", "operation": "project", "question": "Display display_name from exhibits as the sole result.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7e281ac4ee7d2d835606_en", "language": "en", "backend": "mysql", "operation": "lte", "question": "Query inspections for rows where rating is not greater than 1.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE rating <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE rating <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7e1cd79fe94cad9c7a9e_hi", "language": "hi", "backend": "supabase", "operation": "count_eq", "question": "specimens में region Books वाले रिकॉर्ड्स की संख्या दीजिए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT COUNT(*) FROM specimens WHERE region = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT COUNT(*) FROM specimens WHERE region = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "508d2834a4743665db1f_en", "language": "en", "backend": "mysql", "operation": "distinct", "question": "I want to see the distinct kind values available in exhibits.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT DISTINCT kind FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT DISTINCT kind FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "508d2834a4743665db1f_hinglish", "language": "hinglish", "backend": "mysql", "operation": "distinct", "question": "Mujhe exhibits mein available kind ke distinct values dekhne hain.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT DISTINCT kind FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT DISTINCT kind FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7fb4f6e9f1b68e05cfd9_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_desc", "question": "reservations ki poori table salary highest se lowest order mein dikhao.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM reservations ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f9fa9654c0e96fd0477b_en", "language": "en", "backend": "mysql", "operation": "eq", "question": "Bring up all rows from inspections where region has the value Books.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE region = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE region = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f9fa9654c0e96fd0477b_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "eq", "question": "Get the full list of rows in inspections where region is Books.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE region = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE region = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8a2e6d77aa1c3a517ecb_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "having", "question": "Tell me the category groups in exhibits with more than 100 rows and their count values.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT category, COUNT(*) FROM exhibits GROUP BY category HAVING COUNT(*) > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM exhibits GROUP BY category HAVING COUNT(*) > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8a2e6d77aa1c3a517ecb_hinglish", "language": "hinglish", "backend": "mysql", "operation": "having", "question": "Please exhibits me 100 se zyada rows wale category groups aur unki count ko display karo.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT category, COUNT(*) FROM exhibits GROUP BY category HAVING COUNT(*) > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM exhibits GROUP BY category HAVING COUNT(*) > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e62cfd2a6789616a503f_en", "language": "en", "backend": "mysql", "operation": "describe", "question": "Reveal the column structure and types for reservations.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "reservations"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a060620c638d2dc3759e_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project", "question": "display item_name from reservations and nothing else", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c023a5b6f95173ae856d_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gt", "question": "exhibits se un rows ko display karo jahan score 50 se more hai.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE score > 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "affa5e5e14444cd82bb4_en", "language": "en", "backend": "mysql", "operation": "lt", "question": "Query reservations for rows where salary is less than 10.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE salary < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "45c3d3db5419d8a8dfeb_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "or", "question": "Display rows from exhibits where kind is Mumbai or O'Reilly.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'O''Reilly');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'O''Reilly');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d5fc4318c9aedb495460_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lt", "question": "inspections se un rows ko query karo jahan price 50 se kam hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE price < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE price < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "70da7b441b3a995da19a_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "and", "question": "find rows in exhibits where city is O'Reilly and quantity is above 2", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE city = 'O''Reilly' AND quantity > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE city = 'O''Reilly' AND quantity > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4e9184b509f43a710021_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_desc", "question": "retrieve all records from reservations sorted by balance high to low", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations ORDER BY balance DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY balance DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5aaaf945b8fba37eadf4_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_asc", "question": "can i see all rows from inspections sorted by price from small to large", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY price ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5aaaf945b8fba37eadf4_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_asc", "question": "inspections se saara data price ke badhte hue order mein lo aur dikhao.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY price ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a29fdee3339da6711080_hinglish", "language": "hinglish", "backend": "mysql", "operation": "bottom", "question": "rating ke badhte order me specimens ki 20 sabse kam rows, lowest se shuru karke, lao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens ORDER BY rating ASC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY rating ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2e9e0c695ac50e3982d7_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "month", "question": "extract rows from exhibits where order_date month is 12", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE MONTH(order_date) = 12;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE MONTH(order_date) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0294353d6bde2cd5185c_en", "language": "en", "backend": "mysql", "operation": "lt", "question": "Query reservations for rows where cost is less than 1.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE cost < 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE cost < 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "208e9ccf17a923a46aac_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "null", "question": "Show me the rows from inspections where item_name is null.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ba705525122da37a7146_en", "language": "en", "backend": "mysql", "operation": "sort_asc", "question": "View all records in reservations sorted by cost from least to greatest.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY cost ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2ed720de1d1005daa102_en", "language": "en", "backend": "mysql", "operation": "describe", "question": "Reveal the column structure and types for reservations.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "reservations"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "2f545db764e4c6cd353d_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gt", "question": "exhibits se un rows ko display karo jahan salary 0 se more hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE salary > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE salary > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8643d909ff9369a5491c_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "and", "question": "find rows in inspections where kind is Mumbai and rating is above 1000", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE kind = 'Mumbai' AND rating > 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE kind = 'Mumbai' AND rating > 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bfb3f8261ddb2fc2bf28_hinglish", "language": "hinglish", "backend": "mysql", "operation": "bottom", "question": "price ke badhte order me specimens ki 3 sabse kam rows, lowest se shuru karke, lao.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens ORDER BY price ASC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens ORDER BY price ASC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "46b2f9dcb5c441ef3e8e_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_sum", "question": "exhibits me kind ke hisaab se rating ka total do please.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT kind, SUM(rating) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT kind, SUM(rating) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a0bc7565d9d93d546770_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "null", "question": "Show me the rows from inspections where item_name is null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f7257c4a2eded61f5c86_hi", "language": "hi", "backend": "supabase", "operation": "sort_desc", "question": "specimens की पूरी सारणी score के उच्चतम से निम्नतम क्रम में दिखाएँ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b2725cc841b5aaa2fba6_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "and", "question": "find rows in inspections where kind is Electronics and price is above 2", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE kind = 'Electronics' AND price > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE kind = 'Electronics' AND price > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "78fd6e66c9d726084543_en", "language": "en", "backend": "mysql", "operation": "count", "question": "What is the volume of data rows in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT COUNT(*) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7fb4f6e9f1b68e05cfd9_en", "language": "en", "backend": "mysql", "operation": "sort_desc", "question": "Display the entire table reservations sorted by salary from highest to lowest.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM reservations ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "02981b7ccddb9b467260_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_asc", "question": "can i see all rows from exhibits sorted by quantity from small to large", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits ORDER BY quantity ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY quantity ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "02981b7ccddb9b467260_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_asc", "question": "exhibits se saara data quantity ke badhte hue order mein lo aur dikhao.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits ORDER BY quantity ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY quantity ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9f8fa242b083f38bc5f8_hinglish", "language": "hinglish", "backend": "mysql", "operation": "month", "question": "Show me the data from inspections where order_date ka month 4 hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE MONTH(order_date) = 4;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE MONTH(order_date) = 4;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f9fa9654c0e96fd0477b_hinglish", "language": "hinglish", "backend": "mysql", "operation": "eq", "question": "inspections se wo saari rows lao jahan region ka value Books hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE region = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE region = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2c31add7d7b45843dec1_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project_eq", "question": "inspections se display_name ki list do jahan kind Pune hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM inspections WHERE kind = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b02f421828127e2eb96b_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lte", "question": "exhibits se wo rows dhundo jaha price 2 se bada nahi hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price <= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price <= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "475287613764159ee5eb_hi", "language": "hi", "backend": "supabase", "operation": "group_count", "question": "reservations में हर category के लिए पंक्तियों की संख्या खोजें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e62cfd2a6789616a503f_hinglish", "language": "hinglish", "backend": "mysql", "operation": "describe", "question": "Mujhe reservations ke columns aur types ke baare mein info do.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "reservations"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "1695d78b68e4d8fca6f0_hi", "language": "hi", "backend": "supabase", "operation": "and", "question": "exhibits में वे पंक्तियाँ खोजें जहाँ region Delhi है और amount 1500 से ऊपर है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE region = 'Delhi' AND amount > 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE region = 'Delhi' AND amount > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "45c3d3db5419d8a8dfeb_hinglish", "language": "hinglish", "backend": "mysql", "operation": "or", "question": "exhibits mein kind Mumbai ya O'Reilly wali rows dikhao na.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'O''Reilly');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'O''Reilly');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "23c0aa3a4596f995ad2a_hi", "language": "hi", "backend": "mysql", "operation": "count", "question": "specimens में कितनी लाइनें मौजूद हैं?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5890e52f983d16adbae7_en", "language": "en", "backend": "mysql", "operation": "top", "question": "Give me the top 3 rows of inspections sorted by quantity from high to low.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY quantity DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY quantity DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "70da7b441b3a995da19a_en", "language": "en", "backend": "mysql", "operation": "and", "question": "Show me the exhibits rows where city is O'Reilly and quantity exceeds 2.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE city = 'O''Reilly' AND quantity > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE city = 'O''Reilly' AND quantity > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4e9184b509f43a710021_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_desc", "question": "reservations ki poori table balance highest se lowest order mein dikhao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations ORDER BY balance DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY balance DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "55cd623d82474692b1a9_hi", "language": "hi", "backend": "mysql", "operation": "count", "question": "specimens में कितनी लाइनें मौजूद हैं?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens WHERE city = 'Agra';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "no such column: city"}}
{"id": "ba705525122da37a7146_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_asc", "question": "reservations se saara data cost ke badhte hue order mein lo aur dikhao.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY cost ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2ed720de1d1005daa102_hinglish", "language": "hinglish", "backend": "mysql", "operation": "describe", "question": "Mujhe reservations ke columns aur types ke baare mein info do.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "reservations"}}, "output": "{\"action\":\"call\",\"name\":\"warehoation.describe_table\",\"arguments\":{\"table\":\"reservations\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false, "error": "Unknown tool"}}
{"id": "8643d909ff9369a5491c_en", "language": "en", "backend": "mysql", "operation": "and", "question": "Show me the inspections rows where kind is Mumbai and rating exceeds 1000.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE kind = 'Mumbai' AND rating > 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE kind = 'Mumbai' AND rating > 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "30a7ec8a99d4ee6b6d0e_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "date_after", "question": "Query rows in inspections where order_date is 2026-12-15 onwards.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM inspections WHERE order_date >= '2026-12-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE order_date >= '2026-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7e281ac4ee7d2d835606_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lte", "question": "inspections se wo rows dhundo jaha rating 1 se bada nahi hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE rating <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE rating <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b2725cc841b5aaa2fba6_en", "language": "en", "backend": "mysql", "operation": "and", "question": "Show me the inspections rows where kind is Electronics and price exceeds 2.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE kind = 'Electronics' AND price > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE kind = 'Electronics' AND price > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "82c104614df1ce6e2762_hi", "language": "hi", "backend": "supabase", "operation": "gt", "question": "specimens से उन पंक्तियों को प्रदर्शित करें जहाँ salary 100 से ज्यादा है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE salary > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE salary > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c01aaa64706cccb8ac54_en", "language": "en", "backend": "mysql", "operation": "between", "question": "Select all items from specimens where the balance column has a value between 0 and 50, inclusive.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE balance BETWEEN 0 AND 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE balance BETWEEN 0 AND 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6f34909f5b1104bf7ac1_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "or", "question": "Display rows from reservations where region is Delhi or pending.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE region IN ('Delhi', 'pending');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE region IN ('Delhi', 'pending');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6f34909f5b1104bf7ac1_hinglish", "language": "hinglish", "backend": "mysql", "operation": "or", "question": "reservations mein region Delhi ya pending wali rows dikhao na.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE region IN ('Delhi', 'pending');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE region IN ('Delhi', 'pending');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "cd5bb3382b9e6c8604e8_hi", "language": "hi", "backend": "mysql", "operation": "max", "question": "specimens में rating का सबसे बड़ा नंबर क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(rating) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(rating) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "45c3d3db5419d8a8dfeb_en", "language": "en", "backend": "mysql", "operation": "or", "question": "Bring up rows from exhibits where kind is either Mumbai or O'Reilly.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'O''Reilly');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'O''Reilly');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "223765dc6e033f0ab2fd_hi", "language": "hi", "backend": "supabase", "operation": "month", "question": "मुझे specimens से वे डेटा पंक्तियां दें जहाँ created_at का महीना 9 है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE EXTRACT(MONTH FROM created_at) = 9;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT * FROM specimens WHERE EXTRACT(MONTH FROM created_at) = 9;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "00c24a7cc97ee9debf36_hinglish", "language": "hinglish", "backend": "mysql", "operation": "contains", "question": "Output karo exhibits mein title mein pending match hone wali rows, case insensitive raho.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM exhibits WHERE LOWER(title) LIKE LOWER('%pending%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE LOWER(title) LIKE LOWER('%pending%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "49c2e66e48d062e5c69c_hi", "language": "hi", "backend": "mysql", "operation": "write", "question": "specimens का पूरा कंटेंट डिलीट करें।", "expected": {"action": "clarify", "question": "यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "42cda457b51f6f9d3056_en", "language": "en", "backend": "mysql", "operation": "eq", "question": "Bring up all rows from inspections where region has the value O'Reilly.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE region = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE region = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "42cda457b51f6f9d3056_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "eq", "question": "Get the full list of rows in inspections where region is O'Reilly.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE region = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE region = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "04b2e5886e750df09a51_hinglish", "language": "hinglish", "backend": "mysql", "operation": "null", "question": "reservations se display_name null wali rows select karo.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations WHERE display_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4e9184b509f43a710021_en", "language": "en", "backend": "mysql", "operation": "sort_desc", "question": "Display the entire table reservations sorted by balance from highest to lowest.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations ORDER BY balance DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY balance DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2e9e0c695ac50e3982d7_hinglish", "language": "hinglish", "backend": "mysql", "operation": "month", "question": "Show me the data from exhibits where order_date ka month 12 hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE MONTH(order_date) = 12;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE MONTH(order_date) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9d0222589e9977ea3f57_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lte", "question": "show the reservations table rows where balance is at most 1500", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations WHERE balance <= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance <= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "208e9ccf17a923a46aac_en", "language": "en", "backend": "mysql", "operation": "null", "question": "I want to view the rows in inspections where item_name is null.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8f9ab277b699f2cbe988_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_desc", "question": "retrieve all records from reservations sorted by score high to low", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ba705525122da37a7146_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_asc", "question": "can i see all rows from reservations sorted by cost from small to large", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY cost ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2ca6667f987742cde1e7_hi", "language": "hi", "backend": "mysql", "operation": "write", "question": "specimens का पूरा कंटेंट डिलीट करें।", "expected": {"action": "clarify", "question": "यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "41762bd116b975652e00_en", "language": "en", "backend": "mysql", "operation": "eq", "question": "Bring up all rows from reservations where status has the value South.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE status = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE status = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "41762bd116b975652e00_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "eq", "question": "Get the full list of rows in reservations where status is South.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE status = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE status = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "41762bd116b975652e00_hinglish", "language": "hinglish", "backend": "mysql", "operation": "eq", "question": "reservations se wo saari rows lao jahan status ka value South hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE status = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE status = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d0346c74f1cfe556b2cb_hi", "language": "hi", "backend": "mysql", "operation": "project_eq", "question": "inspections से description की सूची दें जहाँ status Books है।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT description FROM inspections WHERE status = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT description FROM inspections WHERE status = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d58041d2866c015bc923_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lte", "question": "show the reservations table rows where amount is at most 100", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE amount <= 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE amount <= 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a0bc7565d9d93d546770_en", "language": "en", "backend": "mysql", "operation": "null", "question": "I want to view the rows in inspections where item_name is null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5f07a4c7c387316664f1_en", "language": "en", "backend": "mysql", "operation": "missing_value", "question": "Execute a filter on exhibits based on department, but I have not specified a value.", "expected": {"action": "clarify", "question": "Which value should department equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should department equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "dd1037fa65f5f5a2a06f_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project_eq", "question": "give me customer_name from reservations where department is North", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT customer_name FROM reservations WHERE department = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT customer_name FROM reservations WHERE department = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e02d95c6aba4ed336a6a_hi", "language": "hi", "backend": "supabase", "operation": "gte", "question": "reservations से वे पंक्तियाँ दिखाएँ जहाँ cost 50 या उससे बड़ा है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE cost >= 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE cost >= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dc59f00f50b6c8f7bc16_en", "language": "en", "backend": "mysql", "operation": "between", "question": "Select all items from exhibits where the amount column has a value between 20 and 520, inclusive.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM exhibits WHERE amount BETWEEN 20 AND 520;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE amount BETWEEN 20 AND 520;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9956c110975a38c3d293_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "top", "question": "I need 3 rows from reservations with cost sorted descending.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations ORDER BY cost DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY cost DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3fdbd39c598af9b9ea10_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "missing_value", "question": "execute filter on reservations for status, but i have not specified a value", "expected": {"action": "clarify", "question": "Which value should status equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should status equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "a060620c638d2dc3759e_en", "language": "en", "backend": "mysql", "operation": "project", "question": "Display item_name from reservations as the sole result.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "affa5e5e14444cd82bb4_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lt", "question": "can you select the rows from reservations where salary is less than 10?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE salary < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "affa5e5e14444cd82bb4_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lt", "question": "reservations se un rows ko query karo jahan salary 10 se kam hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE salary < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "42cda457b51f6f9d3056_hinglish", "language": "hinglish", "backend": "mysql", "operation": "eq", "question": "inspections se wo saari rows lao jahan region ka value O'Reilly hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE region = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE region = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d61a27de98bb7d44f15d_hi", "language": "hi", "backend": "mysql", "operation": "count", "question": "specimens में कितनी लाइनें मौजूद हैं?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens WHERE department = 'tpecimens';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "0294353d6bde2cd5185c_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lt", "question": "can you select the rows from reservations where cost is less than 1?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE cost < 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE cost < 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0294353d6bde2cd5185c_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lt", "question": "reservations se un rows ko query karo jahan cost 1 se kam hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE cost < 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE cost < 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9d0222589e9977ea3f57_en", "language": "en", "backend": "mysql", "operation": "lte", "question": "Query reservations for rows where balance is not greater than 1500.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations WHERE balance <= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance <= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8f9ab277b699f2cbe988_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_desc", "question": "reservations ki poori table score highest se lowest order mein dikhao.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "039280d3483e998ccd1b_hi", "language": "hi", "backend": "supabase", "operation": "group_count", "question": "reservations में हर department के लिए पंक्तियों की संख्या खोजें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT department, COUNT(*) FROM reservations GROUP BY department;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d58041d2866c015bc923_en", "language": "en", "backend": "mysql", "operation": "lte", "question": "Query reservations for rows where amount is not greater than 100.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE amount <= 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE amount <= 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b0f56c14239ba5d0227f_en", "language": "en", "backend": "mysql", "operation": "distinct", "question": "I want to see the distinct region values available in reservations.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT DISTINCT region FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT region FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b0f56c14239ba5d0227f_hinglish", "language": "hinglish", "backend": "mysql", "operation": "distinct", "question": "Mujhe reservations mein available region ke distinct values dekhne hain.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT DISTINCT region FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT region FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f4fb7fafb95b5c8287f5_hi", "language": "hi", "backend": "supabase", "operation": "group_count", "question": "reservations में हर department के लिए पंक्तियों की संख्या खोजें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT department, COUNT(*) FROM reservations GROUP BY department;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "55c82b78d80d85926577_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "date_after", "question": "Query rows in exhibits where start_date is 2026-09-15 onwards.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE start_date >= '2026-09-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE start_date >= '2026-09-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dd1037fa65f5f5a2a06f_en", "language": "en", "backend": "mysql", "operation": "project_eq", "question": "List the customer_name from reservations where department is North.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT customer_name FROM reservations WHERE department = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT customer_name FROM reservations WHERE department = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c01aaa64706cccb8ac54_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "between", "question": "pull the rows from specimens where balance is greater than or equal to 0 and less than or equal to 50", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE balance BETWEEN 0 AND 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE balance BETWEEN 0 AND 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6f34909f5b1104bf7ac1_en", "language": "en", "backend": "mysql", "operation": "or", "question": "Bring up rows from reservations where region is either Delhi or pending.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE region IN ('Delhi', 'pending');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE region IN ('Delhi', 'pending');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e7d754ed36b247db55d3_hi", "language": "hi", "backend": "mysql", "operation": "not_null", "question": "specimens का डेटा दिखाएं जहाँ description null नहीं है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "804d318fe46a0e664840_en", "language": "en", "backend": "mysql", "operation": "sort_asc", "question": "View all records in reservations sorted by quantity from least to greatest.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY quantity ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY quantity ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7d4234476674a04380e6_hi", "language": "hi", "backend": "supabase", "operation": "gte", "question": "reservations से वे पंक्तियाँ दिखाएँ जहाँ score 1500 या उससे बड़ा है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE score >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE score >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c50090341e5d5516a579_en", "language": "en", "backend": "mysql", "operation": "not_null", "question": "Present the data from reservations where customer_name is not null.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations WHERE customer_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE customer_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c50090341e5d5516a579_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "not_null", "question": "Find the rows in reservations where customer_name is not null.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations WHERE customer_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE customer_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c50090341e5d5516a579_hinglish", "language": "hinglish", "backend": "mysql", "operation": "not_null", "question": "reservations ka data present karo jahan customer_name null nahi hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations WHERE customer_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE customer_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bd760f1ca7aff546e852_hinglish", "language": "hinglish", "backend": "mysql", "operation": "bottom", "question": "balance ke badhte order me exhibits ki 5 sabse kam rows, lowest se shuru karke, lao.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits ORDER BY balance ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY balance ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "00c24a7cc97ee9debf36_en", "language": "en", "backend": "mysql", "operation": "contains", "question": "List all rows in exhibits where title is equal to pending in a case-insensitive way.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM exhibits WHERE LOWER(title) LIKE LOWER('%pending%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE LOWER(title) LIKE LOWER('%pending%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c07bcd68edd3354fcd83_hi", "language": "hi", "backend": "mysql", "operation": "all", "question": "specimens के सभी रिकॉर्ड प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "70da7b441b3a995da19a_hinglish", "language": "hinglish", "backend": "mysql", "operation": "and", "question": "exhibits mein wo rows dhundo jahan city O'Reilly hai aur quantity 2 se upar hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE city = 'O''Reilly' AND quantity > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE city = 'O''Reilly' AND quantity > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "04b2e5886e750df09a51_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "null", "question": "Show me the rows from reservations where display_name is null.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations WHERE display_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9d0222589e9977ea3f57_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lte", "question": "reservations se wo rows dhundo jaha balance 1500 se bada nahi hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations WHERE balance <= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance <= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5f0b5d842746b96c126b_en", "language": "en", "backend": "mysql", "operation": "between", "question": "Select all items from exhibits where the price column has a value between 1000 and 1100, inclusive.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price BETWEEN 1000 AND 1100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price BETWEEN 1000 AND 1100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8f9ab277b699f2cbe988_en", "language": "en", "backend": "mysql", "operation": "sort_desc", "question": "Display the entire table reservations sorted by score from highest to lowest.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4e4b83d8af9eb4c5efc8_hinglish", "language": "hinglish", "backend": "mysql", "operation": "bottom", "question": "amount ke badhte order me inspections ki 5 sabse kam rows, lowest se shuru karke, lao.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e8cfdfba296bc283222a_hi", "language": "hi", "backend": "supabase", "operation": "month", "question": "मुझे exhibits से वे डेटा पंक्तियां दें जहाँ start_date का महीना 7 है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE EXTRACT(MONTH FROM start_date) = 7;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE EXTRACT(MONTH FROM start_date) = 7;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2cf634192448e68935c1_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "final", "question": "db tool returned 5 rows. tell me the total.", "expected": {"action": "answer", "text": "The tool returned 5 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 5 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "2cf634192448e68935c1_hinglish", "language": "hinglish", "backend": "mysql", "operation": "final", "question": "Kitni rows aayin database tool se? Count hai 5.", "expected": {"action": "answer", "text": "Tool ne 5 rows return ki."}, "output": "{\"action\":\"answer\",\"text\":\"Tool ne 5 rows return ki.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "db7eb47a2bad512e837d_hi", "language": "hi", "backend": "mysql", "operation": "all", "question": "specimens के सभी रिकॉर्ड प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8643d909ff9369a5491c_hinglish", "language": "hinglish", "backend": "mysql", "operation": "and", "question": "inspections mein wo rows dhundo jahan kind Mumbai hai aur rating 1000 se upar hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE kind = 'Mumbai' AND rating > 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE kind = 'Mumbai' AND rating > 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b10e3bcce6871b79df41_en", "language": "en", "backend": "mysql", "operation": "sort_asc", "question": "View all records in reservations sorted by price from least to greatest.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations ORDER BY price ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d58041d2866c015bc923_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lte", "question": "reservations se wo rows dhundo jaha amount 100 se bada nahi hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE amount <= 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE amount <= 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f97582c94f2100eb1007_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "having", "question": "Tell me the department groups in exhibits with more than 500 rows and their count values.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f97582c94f2100eb1007_hinglish", "language": "hinglish", "backend": "mysql", "operation": "having", "question": "Please exhibits me 500 se zyada rows wale department groups aur unki count ko display karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a59c6e6bcec70a1667e5_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "final", "question": "db tool returned 35 rows. tell me the total.", "expected": {"action": "answer", "text": "The tool returned 35 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 35 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "a59c6e6bcec70a1667e5_hinglish", "language": "hinglish", "backend": "mysql", "operation": "final", "question": "Kitni rows aayin database tool se? Count hai 35.", "expected": {"action": "answer", "text": "Tool ne 35 rows return ki."}, "output": "{\"action\":\"answer\",\"text\":\"Tool ne 35 rows return ki.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "860c064ccde26d474c46_hi", "language": "hi", "backend": "mysql", "operation": "project", "question": "सिर्फ exhibits का display_name दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dd1037fa65f5f5a2a06f_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project_eq", "question": "reservations se customer_name ki list do jahan department North hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT customer_name FROM reservations WHERE department = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT customer_name FROM reservations WHERE department = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dc59f00f50b6c8f7bc16_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "between", "question": "pull the rows from exhibits where amount is greater than or equal to 20 and less than or equal to 520", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM exhibits WHERE amount BETWEEN 20 AND 520;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE amount BETWEEN 20 AND 520;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b2725cc841b5aaa2fba6_hinglish", "language": "hinglish", "backend": "mysql", "operation": "and", "question": "inspections mein wo rows dhundo jahan kind Electronics hai aur price 2 se upar hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE kind = 'Electronics' AND price > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE kind = 'Electronics' AND price > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e71814820a36875f0ee5_hi", "language": "hi", "backend": "mysql", "operation": "not_null", "question": "specimens का डेटा दिखाएं जहाँ description null नहीं है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "804d318fe46a0e664840_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_asc", "question": "reservations se saara data quantity ke badhte hue order mein lo aur dikhao.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY quantity ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY quantity ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8a2e6d77aa1c3a517ecb_en", "language": "en", "backend": "mysql", "operation": "having", "question": "Give me the category groups in exhibits where the row count is higher than 100, along with the count values.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT category, COUNT(*) FROM exhibits GROUP BY category HAVING COUNT(*) > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM exhibits GROUP BY category HAVING COUNT(*) > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7506912696ddf94cfae4_hi", "language": "hi", "backend": "mysql", "operation": "max", "question": "exhibits में amount का सबसे बड़ा नंबर क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(amount) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(amount) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e92015bce168f8f9a90a_hi", "language": "hi", "backend": "supabase", "operation": "sort_desc", "question": "exhibits की पूरी सारणी score के उच्चतम से निम्नतम क्रम में दिखाएँ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4a79981adf7647bac6b9_hi", "language": "hi", "backend": "supabase", "operation": "gt", "question": "exhibits से उन पंक्तियों को प्रदर्शित करें जहाँ amount 500 से ज्यादा है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE amount > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE amount > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d30264bbc3b08aedbc51_hi", "language": "hi", "backend": "mysql", "operation": "sum", "question": "exhibits में score के सभी मानों का योग करें।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT SUM(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fd0bab1cc08d7298f131_hi", "language": "hi", "backend": "supabase", "operation": "top", "question": "शीर्ष 20 पंक्तियाँ exhibits से प्राप्त करें, salary के आधार पर उच्चतम पहले।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "cfb6d9bde68e2cdc5bf9_hi", "language": "hi", "backend": "mysql", "operation": "project_eq", "question": "reservations से name की सूची दें जहाँ category Books है।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT name FROM reservations WHERE category = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT name FROM reservations WHERE category = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8d2022223833a85bd2d1_hi", "language": "hi", "backend": "mysql", "operation": "max", "question": "exhibits में salary का सबसे बड़ा नंबर क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT MAX(salary) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT MAX(salary) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b10e3bcce6871b79df41_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_asc", "question": "reservations se saara data price ke badhte hue order mein lo aur dikhao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations ORDER BY price ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "30a7ec8a99d4ee6b6d0e_en", "language": "en", "backend": "mysql", "operation": "date_after", "question": "Extract rows from inspections where order_date falls on 2026-12-15 or beyond.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM inspections WHERE order_date >= '2026-12-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE order_date >= '2026-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dc59f00f50b6c8f7bc16_hinglish", "language": "hinglish", "backend": "mysql", "operation": "between", "question": "exhibits se wo saare items select karo jahan amount column ka value 20 aur 520 ke beech hai, inclusive.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM exhibits WHERE amount BETWEEN 20 AND 520;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE amount BETWEEN 20 AND 520;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9956c110975a38c3d293_hinglish", "language": "hinglish", "backend": "mysql", "operation": "top", "question": "Top 3 rows reservations se lo, cost ke hisaab se highest first.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations ORDER BY cost DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY cost DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c01aaa64706cccb8ac54_hinglish", "language": "hinglish", "backend": "mysql", "operation": "between", "question": "specimens se wo saare items select karo jahan balance column ka value 0 aur 50 ke beech hai, inclusive.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE balance BETWEEN 0 AND 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE balance BETWEEN 0 AND 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "804d318fe46a0e664840_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_asc", "question": "can i see all rows from reservations sorted by quantity from small to large", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY quantity ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY quantity ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d1f372f544fa10f6df1b_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "join", "question": "output reservations.title and reservations_groups.label by joining reservations.group_id to reservations_groups.id", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT reservations.title, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT reservations.title, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3fdbd39c598af9b9ea10_en", "language": "en", "backend": "mysql", "operation": "missing_value", "question": "Execute a filter on reservations based on status, but I have not specified a value.", "expected": {"action": "clarify", "question": "Which value should status equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should status equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "0d94a075cd7e5759bdfa_en", "language": "en", "backend": "mysql", "operation": "between", "question": "Select all items from exhibits where the rating column has a value between 10 and 110, inclusive.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE rating BETWEEN 10 AND 110;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE rating BETWEEN 10 AND 110;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9aed1992a793fbf97601_hi", "language": "hi", "backend": "mysql", "operation": "min", "question": "exhibits में price का न्यूनतम परिमाण क्या है?", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT MIN(price) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT MIN(price) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "15b265bd8bbdb3515605_hi", "language": "hi", "backend": "mysql", "operation": "project_eq", "question": "inspections से display_name की सूची दें जहाँ kind Electronics है।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT display_name FROM inspections WHERE kind = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT display_name FROM inspections WHERE kind = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "04b2e5886e750df09a51_en", "language": "en", "backend": "mysql", "operation": "null", "question": "I want to view the rows in reservations where display_name is null.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations WHERE display_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5f0b5d842746b96c126b_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "between", "question": "pull the rows from exhibits where price is greater than or equal to 1000 and less than or equal to 1100", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price BETWEEN 1000 AND 1100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price BETWEEN 1000 AND 1100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b10e3bcce6871b79df41_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_asc", "question": "can i see all rows from reservations sorted by price from small to large", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations ORDER BY price ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7a33de7fd7159808b04a_hi", "language": "hi", "backend": "supabase", "operation": "month", "question": "मुझे inspections से वे डेटा पंक्तियां दें जहाँ created_at का महीना 11 है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE EXTRACT(MONTH FROM created_at) = 11;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE EXTRACT(MONTH FROM created_at) = 11;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5f07a4c7c387316664f1_hinglish", "language": "hinglish", "backend": "mysql", "operation": "missing_value", "question": "department ke basis pe exhibits pe ek filter execute karo, lekin maine koi value specify nahi ki hai.", "expected": {"action": "clarify", "question": "department ki kaunsi value chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"department ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "78fd6e66c9d726084543_hi", "language": "hi", "backend": "mysql", "operation": "count", "question": "exhibits में कितनी लाइनें मौजूद हैं?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT COUNT(*) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "adff4eabe2639c0c128c_hi", "language": "hi", "backend": "mysql", "operation": "all", "question": "inspections के सभी रिकॉर्ड प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d1f372f544fa10f6df1b_en", "language": "en", "backend": "supabase", "operation": "join", "question": "Extract reservations.title and reservations_groups.label by joining reservations.group_id to reservations_groups.id.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT reservations.title, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT reservations.title, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2162ee1266aed9320937_hi", "language": "hi", "backend": "mysql", "operation": "write", "question": "inspections का पूरा कंटेंट डिलीट करें।", "expected": {"action": "clarify", "question": "यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "5d97abdb52189efbe117_hi", "language": "hi", "backend": "mysql", "operation": "group_sum", "question": "exhibits में status के अनुसार amount का कुल कृपया दें।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT status, SUM(amount) FROM exhibits GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT status, SUM(amount) FROM exhibits GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "916ff7ce3eea9aa64393_hi", "language": "hi", "backend": "supabase", "operation": "missing_value", "question": "kind के आधार पर reservations पर एक फ़िल्टर निष्पादित करें, लेकिन मैंने कोई मान निर्दिष्ट नहीं किया है।", "expected": {"action": "clarify", "question": "kind का कौन सा मान चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"kind का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "6f28c044459ce99cf405_hi", "language": "hi", "backend": "mysql", "operation": "max", "question": "exhibits में quantity का सबसे बड़ा नंबर क्या है?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT MAX(quantity) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT MAX(quantity) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6d9e81a9f52b8d33060a_hi", "language": "hi", "backend": "mysql", "operation": "all", "question": "exhibits के सभी रिकॉर्ड प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5f0b5d842746b96c126b_hinglish", "language": "hinglish", "backend": "mysql", "operation": "between", "question": "exhibits se wo saare items select karo jahan price column ka value 1000 aur 1100 ke beech hai, inclusive.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price BETWEEN 1000 AND 1100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price BETWEEN 1000 AND 1100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2cf634192448e68935c1_en", "language": "en", "backend": "mysql", "operation": "final", "question": "The database tool returned 5 rows. Can you tell me the number?", "expected": {"action": "answer", "text": "The tool returned 5 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 5 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "ccbb8ac5e43114f43047_hi", "language": "hi", "backend": "supabase", "operation": "between", "question": "exhibits से वे सभी आइटम चुनें जहाँ balance कॉलम का मान 50 और 100 के बीच है, समावेशी।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE balance BETWEEN 50 AND 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "'query'"}}
{"id": "c466f263014e2eb31cf5_hi", "language": "hi", "backend": "mysql", "operation": "max", "question": "exhibits में rating का सबसे बड़ा नंबर क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT MAX(rating) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT MAX(rating) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6dd9f03c26298630777f_hi", "language": "hi", "backend": "supabase", "operation": "between", "question": "inspections से वे सभी आइटम चुनें जहाँ score कॉलम का मान 1 और 101 के बीच है, समावेशी।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE score BETWEEN 1 AND 101;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_inspections\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "'query'"}}
{"id": "feb75e02c031a950e223_hi", "language": "hi", "backend": "mysql", "operation": "group_sum", "question": "inspections में category के अनुसार cost का कुल कृपया दें।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT category, SUM(cost) FROM inspections GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT category, SUM(cost) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "55c82b78d80d85926577_en", "language": "en", "backend": "mysql", "operation": "date_after", "question": "Extract rows from exhibits where start_date falls on 2026-09-15 or beyond.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE start_date >= '2026-09-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE start_date >= '2026-09-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a59c6e6bcec70a1667e5_en", "language": "en", "backend": "mysql", "operation": "final", "question": "The database tool returned 35 rows. Can you tell me the number?", "expected": {"action": "answer", "text": "The tool returned 35 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 35 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "a6079b58d4365a8f70c8_hi", "language": "hi", "backend": "mysql", "operation": "null", "question": "inspections से title null वाली पंक्तियाँ चुनें।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9956c110975a38c3d293_en", "language": "en", "backend": "mysql", "operation": "top", "question": "Give me the top 3 rows of reservations sorted by cost from high to low.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations ORDER BY cost DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY cost DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2c31add7d7b45843dec1_hi", "language": "hi", "backend": "mysql", "operation": "project_eq", "question": "inspections से display_name की सूची दें जहाँ kind Pune है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM inspections WHERE kind = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a060620c638d2dc3759e_hi", "language": "hi", "backend": "mysql", "operation": "project", "question": "सिर्फ reservations का item_name दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0d94a075cd7e5759bdfa_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "between", "question": "pull the rows from exhibits where rating is greater than or equal to 10 and less than or equal to 110", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE rating BETWEEN 10 AND 110;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE rating BETWEEN 10 AND 110;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "01f4f9527d44f09ab6b6_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "having", "question": "Tell me the kind groups in reservations with more than 200 rows and their count values.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM reservations GROUP BY kind HAVING COUNT(*) > 200;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM reservations GROUP BY kind HAVING COUNT(*) > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "01f4f9527d44f09ab6b6_hinglish", "language": "hinglish", "backend": "mysql", "operation": "having", "question": "Please reservations me 200 se zyada rows wale kind groups aur unki count ko display karo.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM reservations GROUP BY kind HAVING COUNT(*) > 200;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM reservations GROUP BY kind HAVING COUNT(*) > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "139a72e10281049d1fed_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "join", "question": "output specimens.name and specimens_groups.label by joining specimens.group_id to specimens_groups.id", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT specimens.name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT specimens.name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c0b9cf3b44d96839f256_hi", "language": "hi", "backend": "mysql", "operation": "min", "question": "inspections में balance का न्यूनतम परिमाण क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(balance) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f55358178d003de7ba56_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "final", "question": "db tool returned 133 rows. tell me the total.", "expected": {"action": "answer", "text": "The tool returned 133 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 133 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "f55358178d003de7ba56_hinglish", "language": "hinglish", "backend": "mysql", "operation": "final", "question": "Kitni rows aayin database tool se? Count hai 133.", "expected": {"action": "answer", "text": "Tool ne 133 rows return ki."}, "output": "{\"action\":\"answer\",\"text\":\"Tool ne 133 rows return ki.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "bc8593644a72fe8b9c78_hi", "language": "hi", "backend": "mysql", "operation": "min", "question": "exhibits में score का न्यूनतम परिमाण क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6c37ae504c7cb31a6ef3_hi", "language": "hi", "backend": "mysql", "operation": "list_tables", "question": "इस डेटाबेस की तालिकाओं का विवरण दीजिए।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6972a0b38fd4a8758723_hi", "language": "hi", "backend": "mysql", "operation": "group_sum", "question": "exhibits में kind के अनुसार score का कुल कृपया दें।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT kind, SUM(score) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT kind, SUM(score) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0d94a075cd7e5759bdfa_hinglish", "language": "hinglish", "backend": "mysql", "operation": "between", "question": "exhibits se wo saare items select karo jahan rating column ka value 10 aur 110 ke beech hai, inclusive.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE rating BETWEEN 10 AND 110;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE rating BETWEEN 10 AND 110;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "139a72e10281049d1fed_en", "language": "en", "backend": "mysql", "operation": "join", "question": "Extract specimens.name and specimens_groups.label by joining specimens.group_id to specimens_groups.id.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT specimens.name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT specimens.name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "40ffef08b12e4a3276e3_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "final", "question": "db tool returned 116 rows. tell me the total.", "expected": {"action": "answer", "text": "The tool returned 116 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 116 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "40ffef08b12e4a3276e3_hinglish", "language": "hinglish", "backend": "mysql", "operation": "final", "question": "Kitni rows aayin database tool se? Count hai 116.", "expected": {"action": "answer", "text": "Tool ne 116 rows return ki."}, "output": "{\"action\":\"answer\",\"text\":\"Tool ne 116 rows return ki.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "b7703ebecadd9433c5f8_hi", "language": "hi", "backend": "supabase", "operation": "top", "question": "शीर्ष 3 पंक्तियाँ reservations से प्राप्त करें, quantity के आधार पर उच्चतम पहले।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations ORDER BY quantity DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations ORDER BY quantity DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d96328fdb5748b255271_hi", "language": "hi", "backend": "mysql", "operation": "group_sum", "question": "inspections में category के अनुसार salary का कुल कृपया दें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT category, SUM(salary) FROM inspections GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT category, SUM(salary) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e02df083b1ff1b7fbcf8_hi", "language": "hi", "backend": "supabase", "operation": "join_filter", "question": "कृपया exhibits.customer_name दिखाएँ जहाँ exhibits exhibits.group_id के माध्यम से exhibits_groups से जुड़ा है और exhibits_groups.label North है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT exhibits.customer_name FROM exhibits JOIN exhibits_groups ON exhibits.group_id = exhibits_groups.id WHERE exhibits_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT exhibits.customer_name FROM exhibits JOIN exhibits_groups ON exhibits.group_id = exhibits_groups.id WHERE exhibits_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "30a7ec8a99d4ee6b6d0e_hinglish", "language": "hinglish", "backend": "mysql", "operation": "date_after", "question": "inspections mein se wo rows chuno jinke order_date mein 2026-12-15 ya uske baad ki date hai.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM inspections WHERE order_date >= '2026-12-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE order_date >= '2026-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "eab95ae98810f2228ba6_hi", "language": "hi", "backend": "mysql", "operation": "year", "question": "specimens में start_date में 2026 वाले रिकॉर्ड दिखाएं।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens WHERE YEAR(start_date) = 2026;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE YEAR(start_date) = 2026;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9bb888f912ca90f30c9a_hi", "language": "hi", "backend": "supabase", "operation": "top", "question": "शीर्ष 20 पंक्तियाँ reservations से प्राप्त करें, rating के आधार पर उच्चतम पहले।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations ORDER BY rating DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations ORDER BY rating DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f97582c94f2100eb1007_en", "language": "en", "backend": "mysql", "operation": "having", "question": "Give me the department groups in exhibits where the row count is higher than 500, along with the count values.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "785d62b3d5e4932f5c33_hi", "language": "hi", "backend": "supabase", "operation": "bottom", "question": "rating के बढ़ते क्रम में inspections की 20 सबसे कम पंक्तियां, सबसे कम से शुरू होकर, लें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections ORDER BY rating ASC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections ORDER BY rating ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d1f372f544fa10f6df1b_hinglish", "language": "hinglish", "backend": "supabase", "operation": "join", "question": "Kripya reservations.group_id ko reservations_groups.id se join karke reservations.title aur reservations_groups.label retrieve karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT reservations.title, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT reservations.title, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c4210296305105ab7472_hi", "language": "hi", "backend": "mysql", "operation": "ambiguous", "question": "specimens में उपलब्ध सर्वोत्तम रिकॉर्ड प्रिंट करें।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "3fdbd39c598af9b9ea10_hinglish", "language": "hinglish", "backend": "mysql", "operation": "missing_value", "question": "status ke basis pe reservations pe ek filter execute karo, lekin maine koi value specify nahi ki hai.", "expected": {"action": "clarify", "question": "status ki kaunsi value chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"status ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "8a3194b9f492b50b8100_hi", "language": "hi", "backend": "mysql", "operation": "distinct", "question": "मुझे specimens में उपलब्ध category के विशिष्ट मान देखने हैं।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT category FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3b3a8e441f0ae5cbaad4_hi", "language": "hi", "backend": "mysql", "operation": "ambiguous", "question": "specimens में उपलब्ध सर्वोत्तम रिकॉर्ड प्रिंट करें।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "69ac9800bd7a3ee23ccc_hi", "language": "hi", "backend": "mysql", "operation": "distinct", "question": "मुझे specimens में उपलब्ध category के विशिष्ट मान देखने हैं।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT category FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9bf8948becf74e81a103_hi", "language": "hi", "backend": "mysql", "operation": "list_tables", "question": "इस डेटाबेस की तालिकाओं का विवरण दीजिए।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "844b55044ac74adf4493_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "join", "question": "output reservations.item_name and reservations_groups.label by joining reservations.group_id to reservations_groups.id", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dddd09f34c0e849282a8_hi", "language": "hi", "backend": "supabase", "operation": "gt", "question": "reservations से उन पंक्तियों को प्रदर्शित करें जहाँ salary 20 से ज्यादा है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE salary > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE salary > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1d7ab92421915fd47ebe_hi", "language": "hi", "backend": "mysql", "operation": "year", "question": "inspections में created_at में 2024 वाले रिकॉर्ड दिखाएं।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections WHERE YEAR(created_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE YEAR(created_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9ebca064e63b0347236b_hi", "language": "hi", "backend": "supabase", "operation": "having", "question": "कृपया specimens में 5 से अधिक पंक्तियों वाले region समूहों और उनकी गणना को प्रदर्शित करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "f55358178d003de7ba56_en", "language": "en", "backend": "mysql", "operation": "final", "question": "The database tool returned 133 rows. Can you tell me the number?", "expected": {"action": "answer", "text": "The tool returned 133 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 133 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "208e9ccf17a923a46aac_hi", "language": "hi", "backend": "mysql", "operation": "null", "question": "inspections से item_name null वाली पंक्तियाँ चुनें।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "46b2f9dcb5c441ef3e8e_hi", "language": "hi", "backend": "mysql", "operation": "group_sum", "question": "exhibits में kind के अनुसार rating का कुल कृपया दें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT kind, SUM(rating) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT kind, SUM(rating) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bd5fbea0dbbb3414c53c_hi", "language": "hi", "backend": "supabase", "operation": "having", "question": "कृपया specimens में 0 से अधिक पंक्तियों वाले region समूहों और उनकी गणना को प्रदर्शित करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "a0bc7565d9d93d546770_hi", "language": "hi", "backend": "mysql", "operation": "null", "question": "inspections से item_name null वाली पंक्तियाँ चुनें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "55c82b78d80d85926577_hinglish", "language": "hinglish", "backend": "mysql", "operation": "date_after", "question": "exhibits mein se wo rows chuno jinke start_date mein 2026-09-15 ya uske baad ki date hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE start_date >= '2026-09-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE start_date >= '2026-09-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "844b55044ac74adf4493_en", "language": "en", "backend": "supabase", "operation": "join", "question": "Extract reservations.item_name and reservations_groups.label by joining reservations.group_id to reservations_groups.id.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dd1037fa65f5f5a2a06f_hi", "language": "hi", "backend": "mysql", "operation": "project_eq", "question": "reservations से customer_name की सूची दें जहाँ department North है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT customer_name FROM reservations WHERE department = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT customer_name FROM reservations WHERE department = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d8f9a887c52e766a8e14_hi", "language": "hi", "backend": "mysql", "operation": "year", "question": "exhibits में start_date में 2023 वाले रिकॉर्ड दिखाएं।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE YEAR(start_date) = 2023;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE YEAR(start_date) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "139a72e10281049d1fed_hinglish", "language": "hinglish", "backend": "mysql", "operation": "join", "question": "Kripya specimens.group_id ko specimens_groups.id se join karke specimens.name aur specimens_groups.label retrieve karo.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT specimens.name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT specimens.name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "40ffef08b12e4a3276e3_en", "language": "en", "backend": "mysql", "operation": "final", "question": "The database tool returned 116 rows. Can you tell me the number?", "expected": {"action": "answer", "text": "The tool returned 116 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 116 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "e114d1f216088c4302b2_hi", "language": "hi", "backend": "mysql", "operation": "distinct", "question": "मुझे specimens में उपलब्ध department के विशिष्ट मान देखने हैं।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT DISTINCT department FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT department FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "28cb6c4bf0d7caac91c9_hi", "language": "hi", "backend": "mysql", "operation": "or", "question": "exhibits में kind Mumbai या Electronics वाली पंक्तियाँ दिखाइए।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'Electronics');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a76838f987782ef2d2ed_hi", "language": "hi", "backend": "mysql", "operation": "describe", "question": "टेबल inspections के कॉलम संरचना और टाइप्स दिखाइए।", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "inspections"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "01f4f9527d44f09ab6b6_en", "language": "en", "backend": "mysql", "operation": "having", "question": "Give me the kind groups in reservations where the row count is higher than 200, along with the count values.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM reservations GROUP BY kind HAVING COUNT(*) > 200;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM reservations GROUP BY kind HAVING COUNT(*) > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d09477cb849c0af3827d_hi", "language": "hi", "backend": "mysql", "operation": "ambiguous", "question": "exhibits में उपलब्ध सर्वोत्तम रिकॉर्ड प्रिंट करें।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "253e89e4cdbbd5c2d0f9_hi", "language": "hi", "backend": "mysql", "operation": "group_count", "question": "specimens में हर status के लिए पंक्तियों की संख्या खोजें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT status, COUNT(*) FROM specimens GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT status, COUNT(*) FROM specimens GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a30eeaf286c79c95a1cd_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "join_filter", "question": "show me inspections.item_name for inspections entries where inspections.group_id links to inspections_groups.id and inspections_groups.label is O'Reilly", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT inspections.item_name FROM inspections JOIN inspections_groups ON inspections.group_id = inspections_groups.id WHERE inspections_groups.label = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT inspections.item_name FROM inspections JOIN inspections_groups ON inspections.group_id = inspections_groups.id WHERE inspections_groups.label = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a30eeaf286c79c95a1cd_hinglish", "language": "hinglish", "backend": "supabase", "operation": "join_filter", "question": "Please inspections.item_name dikhao jahan inspections inspections.group_id ke through inspections_groups se juda hai aur inspections_groups.label O'Reilly hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT inspections.item_name FROM inspections JOIN inspections_groups ON inspections.group_id = inspections_groups.id WHERE inspections_groups.label = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT inspections.item_name FROM inspections JOIN inspections_groups ON inspections.group_id = inspections_groups.id WHERE inspections_groups.label = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e62cfd2a6789616a503f_hi", "language": "hi", "backend": "mysql", "operation": "describe", "question": "टेबल reservations के कॉलम संरचना और टाइप्स दिखाइए।", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "reservations"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "c50090341e5d5516a579_hi", "language": "hi", "backend": "mysql", "operation": "not_null", "question": "reservations का डेटा दिखाएं जहाँ customer_name null नहीं है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations WHERE customer_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE customer_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "04b2e5886e750df09a51_hi", "language": "hi", "backend": "mysql", "operation": "null", "question": "reservations से display_name null वाली पंक्तियाँ चुनें।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations WHERE display_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "307d5b7bd93949f7381a_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "join", "question": "output specimens.customer_name and specimens_groups.label by joining specimens.group_id to specimens_groups.id", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2ed720de1d1005daa102_hi", "language": "hi", "backend": "mysql", "operation": "describe", "question": "टेबल reservations के कॉलम संरचना और टाइप्स दिखाइए।", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "reservations"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "aa867c871299c26d5402_hi", "language": "hi", "backend": "mysql", "operation": "lt", "question": "specimens से उन पंक्तियों का क्वेरी करें जहाँ rating 1000 से कम है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE rating < 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE rating < 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3fdbfd3a4d0df901f143_hi", "language": "hi", "backend": "mysql", "operation": "gte", "question": "specimens से वे पंक्तियाँ दिखाएँ जहाँ amount 200 या उससे बड़ा है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE amount >= 200;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE amount >= 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "53220f60789b4f5e4046_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "join", "question": "output specimens.customer_name and specimens_groups.label by joining specimens.group_id to specimens_groups.id", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "19d562dac8a4125562f6_hi", "language": "hi", "backend": "mysql", "operation": "eq", "question": "inspections से वे सभी पंक्तियाँ लाएँ जहाँ city का मान Delhi है।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM inspections WHERE city = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE city = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "844b55044ac74adf4493_hinglish", "language": "hinglish", "backend": "supabase", "operation": "join", "question": "Kripya reservations.group_id ko reservations_groups.id se join karke reservations.item_name aur reservations_groups.label retrieve karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a30eeaf286c79c95a1cd_en", "language": "en", "backend": "supabase", "operation": "join_filter", "question": "Give me inspections.item_name for inspections entries where inspections.group_id points to inspections_groups.id and inspections_groups.label is O'Reilly.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT inspections.item_name FROM inspections JOIN inspections_groups ON inspections.group_id = inspections_groups.id WHERE inspections_groups.label = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT inspections.item_name FROM inspections JOIN inspections_groups ON inspections.group_id = inspections_groups.id WHERE inspections_groups.label = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "30dd94e63a065d87109e_hi", "language": "hi", "backend": "mysql", "operation": "sql_error", "question": "पिछला क्वेरी विफल रहा क्योंकि एक कॉलम अज्ञात था। दोबारा प्रयास करने से पहले specimens स्कीमा की पुष्टि करें।", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "specimens"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"saarmens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "2164b376f732a1c371ab_hi", "language": "hi", "backend": "mysql", "operation": "group_count", "question": "specimens में हर kind के लिए पंक्तियों की संख्या खोजें।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM specimens GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0352841d74f7bd7955ea_hi", "language": "hi", "backend": "mysql", "operation": "lte", "question": "inspections से वे पंक्तियां खोजें जहां score 3 से बड़ा नहीं है।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections WHERE score <= 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE score <= 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "307d5b7bd93949f7381a_en", "language": "en", "backend": "mysql", "operation": "join", "question": "Extract specimens.customer_name and specimens_groups.label by joining specimens.group_id to specimens_groups.id.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "375b86b5cd0d00cd7868_hi", "language": "hi", "backend": "supabase", "operation": "having", "question": "कृपया exhibits में 2 से अधिक पंक्तियों वाले department समूहों और उनकी गणना को प्रदर्शित करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "53220f60789b4f5e4046_en", "language": "en", "backend": "mysql", "operation": "join", "question": "Extract specimens.customer_name and specimens_groups.label by joining specimens.group_id to specimens_groups.id.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6f162db917ded36d0df0_hi", "language": "hi", "backend": "mysql", "operation": "sql_error", "question": "पिछला क्वेरी विफल रहा क्योंकि एक कॉलम अज्ञात था। दोबारा प्रयास करने से पहले specimens स्कीमा की पुष्टि करें।", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "specimens"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"specmens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "1523073b9b419fc60ded_hi", "language": "hi", "backend": "mysql", "operation": "count_eq", "question": "specimens में city Books वाले रिकॉर्ड्स की संख्या दीजिए।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT COUNT(*) FROM specimens WHERE city = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens WHERE city = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9eee465dd6b4c3a090c6_hi", "language": "hi", "backend": "mysql", "operation": "eq", "question": "specimens से वे सभी पंक्तियाँ लाएँ जहाँ region का मान Pune है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE region = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE region = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "cab55a7429427fbc5093_hi", "language": "hi", "backend": "mysql", "operation": "distinct", "question": "मुझे inspections में उपलब्ध department के विशिष्ट मान देखने हैं।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT department FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "45c3d3db5419d8a8dfeb_hi", "language": "hi", "backend": "mysql", "operation": "or", "question": "exhibits में kind Mumbai या O'Reilly वाली पंक्तियाँ दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'O''Reilly');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'O''Reilly');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e2b48c4ec5252934cb20_hi", "language": "hi", "backend": "mysql", "operation": "count_eq", "question": "inspections में city pending वाले रिकॉर्ड्स की संख्या दीजिए।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT COUNT(*) FROM inspections WHERE city = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM inspections WHERE city = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "508d2834a4743665db1f_hi", "language": "hi", "backend": "mysql", "operation": "distinct", "question": "मुझे exhibits में उपलब्ध kind के विशिष्ट मान देखने हैं।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT DISTINCT kind FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT DISTINCT kind FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b02f421828127e2eb96b_hi", "language": "hi", "backend": "mysql", "operation": "lte", "question": "exhibits से वे पंक्तियां खोजें जहां price 2 से बड़ा नहीं है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price <= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price <= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6f34909f5b1104bf7ac1_hi", "language": "hi", "backend": "mysql", "operation": "or", "question": "reservations में region Delhi या pending वाली पंक्तियाँ दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE region IN ('Delhi', 'pending');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE region IN ('Delhi', 'pending');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d5fc4318c9aedb495460_hi", "language": "hi", "backend": "mysql", "operation": "lt", "question": "inspections से उन पंक्तियों का क्वेरी करें जहाँ price 50 से कम है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE price < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE price < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e8f90532d34c12209a83_hi", "language": "hi", "backend": "mysql", "operation": "gte", "question": "inspections से वे पंक्तियाँ दिखाएँ जहाँ salary 1500 या उससे बड़ा है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE salary >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8439e708b96823b39172_hi", "language": "hi", "backend": "mysql", "operation": "sort_asc", "question": "inspections से सभी डेटा amount के बढ़ते हुए क्रम में लें और दिखाएं।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7e281ac4ee7d2d835606_hi", "language": "hi", "backend": "mysql", "operation": "lte", "question": "inspections से वे पंक्तियां खोजें जहां rating 1 से बड़ा नहीं है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE rating <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE rating <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f9fa9654c0e96fd0477b_hi", "language": "hi", "backend": "mysql", "operation": "eq", "question": "inspections से वे सभी पंक्तियाँ लाएँ जहाँ region का मान Books है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE region = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE region = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "81b12886ddb859bb71db_hi", "language": "hi", "backend": "mysql", "operation": "count_eq", "question": "inspections में city pending वाले रिकॉर्ड्स की संख्या दीजिए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM inspections WHERE city = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM inspections WHERE city = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "35b25e2b46b13ad30caf_hi", "language": "hi", "backend": "mysql", "operation": "ambiguous", "question": "reservations में उपलब्ध सर्वोत्तम रिकॉर्ड प्रिंट करें।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "307d5b7bd93949f7381a_hinglish", "language": "hinglish", "backend": "mysql", "operation": "join", "question": "Kripya specimens.group_id ko specimens_groups.id se join karke specimens.customer_name aur specimens_groups.label retrieve karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "24661fe309a7c9930145_hi", "language": "hi", "backend": "mysql", "operation": "month", "question": "मुझे specimens से वे डेटा पंक्तियां दें जहाँ recorded_at का महीना 1 है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE MONTH(recorded_at) = 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE MONTH(recorded_at) = 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "53220f60789b4f5e4046_hinglish", "language": "hinglish", "backend": "mysql", "operation": "join", "question": "Kripya specimens.group_id ko specimens_groups.id se join karke specimens.customer_name aur specimens_groups.label retrieve karo.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "63fa94106f0cf26f7da2_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "join_filter", "question": "show me specimens.title for specimens entries where specimens.group_id links to specimens_groups.id and specimens_groups.label is North", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT specimens.title FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id WHERE specimens_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.title FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id WHERE specimens_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "63fa94106f0cf26f7da2_hinglish", "language": "hinglish", "backend": "mysql", "operation": "join_filter", "question": "Please specimens.title dikhao jahan specimens specimens.group_id ke through specimens_groups se juda hai aur specimens_groups.label North hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT specimens.title FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id WHERE specimens_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.title FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id WHERE specimens_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1d82ef2b9527e797c466_hi", "language": "hi", "backend": "supabase", "operation": "date_after", "question": "inspections में से वे पंक्तियाँ चुनें जिनके order_date में 2022-04-15 या उससे बाद की तारीख है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE order_date >= '2022-04-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE order_date >= '2022-04-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "70da7b441b3a995da19a_hi", "language": "hi", "backend": "mysql", "operation": "and", "question": "exhibits में वे पंक्तियाँ खोजें जहाँ city O'Reilly है और quantity 2 से ऊपर है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE city = 'O''Reilly' AND quantity > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE city = 'O''Reilly' AND quantity > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5aaaf945b8fba37eadf4_hi", "language": "hi", "backend": "mysql", "operation": "sort_asc", "question": "inspections से सभी डेटा price के बढ़ते हुए क्रम में लें और दिखाएं।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY price ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f1f8414c2d47f5396d69_hi", "language": "hi", "backend": "supabase", "operation": "date_after", "question": "exhibits में से वे पंक्तियाँ चुनें जिनके created_at में 2023-03-15 या उससे बाद की तारीख है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE created_at >= '2023-03-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE created_at >= '2023-03-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "41762bd116b975652e00_hi", "language": "hi", "backend": "mysql", "operation": "eq", "question": "reservations से वे सभी पंक्तियाँ लाएँ जहाँ status का मान South है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE status = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE status = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8643d909ff9369a5491c_hi", "language": "hi", "backend": "mysql", "operation": "and", "question": "inspections में वे पंक्तियाँ खोजें जहाँ kind Mumbai है और rating 1000 से ऊपर है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE kind = 'Mumbai' AND rating > 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE kind = 'Mumbai' AND rating > 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4c5f91f7793bd0200f31_hi", "language": "hi", "backend": "mysql", "operation": "top", "question": "शीर्ष 1 पंक्तियाँ specimens से प्राप्त करें, amount के आधार पर उच्चतम पहले।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens ORDER BY amount DESC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY amount DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "63fa94106f0cf26f7da2_en", "language": "en", "backend": "mysql", "operation": "join_filter", "question": "Give me specimens.title for specimens entries where specimens.group_id points to specimens_groups.id and specimens_groups.label is North.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT specimens.title FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id WHERE specimens_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.title FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id WHERE specimens_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b2725cc841b5aaa2fba6_hi", "language": "hi", "backend": "mysql", "operation": "and", "question": "inspections में वे पंक्तियाँ खोजें जहाँ kind Electronics है और price 2 से ऊपर है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE kind = 'Electronics' AND price > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE kind = 'Electronics' AND price > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "02981b7ccddb9b467260_hi", "language": "hi", "backend": "mysql", "operation": "sort_asc", "question": "exhibits से सभी डेटा quantity के बढ़ते हुए क्रम में लें और दिखाएं।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits ORDER BY quantity ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY quantity ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "797b9b914530f2eb502a_hi", "language": "hi", "backend": "supabase", "operation": "final", "question": "डेटाबेस टूल ने 48 पंक्तियां वापस कीं। कितनी पंक्तियां लौटीं?", "expected": {"action": "answer", "text": "टूल ने 48 पंक्तियाँ लौटाईं।"}, "output": "{\"action\":\"answer\",\"text\":\"टूल ने 48 पंक्तियाँ लौटाईं।\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "affa5e5e14444cd82bb4_hi", "language": "hi", "backend": "mysql", "operation": "lt", "question": "reservations से उन पंक्तियों का क्वेरी करें जहाँ salary 10 से कम है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE salary < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "58eb6ab2fb9c69becdbd_hi", "language": "hi", "backend": "supabase", "operation": "date_after", "question": "reservations में से वे पंक्तियाँ चुनें जिनके created_at में 2022-08-15 या उससे बाद की तारीख है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE created_at >= '2022-08-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE created_at >= '2022-08-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "42cda457b51f6f9d3056_hi", "language": "hi", "backend": "mysql", "operation": "eq", "question": "inspections से वे सभी पंक्तियाँ लाएँ जहाँ region का मान O'Reilly है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE region = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE region = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0294353d6bde2cd5185c_hi", "language": "hi", "backend": "mysql", "operation": "lt", "question": "reservations से उन पंक्तियों का क्वेरी करें जहाँ cost 1 से कम है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE cost < 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE cost < 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9d0222589e9977ea3f57_hi", "language": "hi", "backend": "mysql", "operation": "lte", "question": "reservations से वे पंक्तियां खोजें जहां balance 1500 से बड़ा नहीं है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations WHERE balance <= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance <= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ba705525122da37a7146_hi", "language": "hi", "backend": "mysql", "operation": "sort_asc", "question": "reservations से सभी डेटा cost के बढ़ते हुए क्रम में लें और दिखाएं।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY cost ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d58041d2866c015bc923_hi", "language": "hi", "backend": "mysql", "operation": "lte", "question": "reservations से वे पंक्तियां खोजें जहां amount 100 से बड़ा नहीं है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE amount <= 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE amount <= 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b0f56c14239ba5d0227f_hi", "language": "hi", "backend": "mysql", "operation": "distinct", "question": "मुझे reservations में उपलब्ध region के विशिष्ट मान देखने हैं।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT DISTINCT region FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT region FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b9c176c07809f42564a0_hi", "language": "hi", "backend": "mysql", "operation": "top", "question": "शीर्ष 5 पंक्तियाँ inspections से प्राप्त करें, amount के आधार पर उच्चतम पहले।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b45bf4e81303e2b4da0f_hi", "language": "hi", "backend": "mysql", "operation": "gt", "question": "specimens से उन पंक्तियों को प्रदर्शित करें जहाँ price 1 से ज्यादा है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens WHERE price > 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE price > 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6519cd640e9e4ab22a6e_hi", "language": "hi", "backend": "mysql", "operation": "gt", "question": "specimens से उन पंक्तियों को प्रदर्शित करें जहाँ score 20 से ज्यादा है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE score > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE score > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7fb4f6e9f1b68e05cfd9_hi", "language": "hi", "backend": "mysql", "operation": "sort_desc", "question": "reservations की पूरी सारणी salary के उच्चतम से निम्नतम क्रम में दिखाएँ।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM reservations ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "41ea5651dda516a4ac30_hi", "language": "hi", "backend": "supabase", "operation": "date_after", "question": "reservations में से वे पंक्तियाँ चुनें जिनके order_date में 2023-12-15 या उससे बाद की तारीख है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE order_date >= '2023-12-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE order_date >= '2023-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9f8fa242b083f38bc5f8_hi", "language": "hi", "backend": "mysql", "operation": "month", "question": "मुझे inspections से वे डेटा पंक्तियां दें जहाँ order_date का महीना 4 है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE MONTH(order_date) = 4;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE MONTH(order_date) = 4;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c023a5b6f95173ae856d_hi", "language": "hi", "backend": "mysql", "operation": "gt", "question": "exhibits से उन पंक्तियों को प्रदर्शित करें जहाँ score 50 से ज्यादा है।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE score > 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5890e52f983d16adbae7_hi", "language": "hi", "backend": "mysql", "operation": "top", "question": "शीर्ष 3 पंक्तियाँ inspections से प्राप्त करें, quantity के आधार पर उच्चतम पहले।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY quantity DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY quantity DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a29fdee3339da6711080_hi", "language": "hi", "backend": "mysql", "operation": "bottom", "question": "rating के बढ़ते क्रम में specimens की 20 सबसे कम पंक्तियां, सबसे कम से शुरू होकर, लें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens ORDER BY rating ASC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY rating ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ae26d8175253b650bb6b_hi", "language": "hi", "backend": "supabase", "operation": "missing_value", "question": "region के आधार पर specimens पर एक फ़िल्टर निष्पादित करें, लेकिन मैंने कोई मान निर्दिष्ट नहीं किया है।", "expected": {"action": "clarify", "question": "region का कौन सा मान चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"region का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "2f545db764e4c6cd353d_hi", "language": "hi", "backend": "mysql", "operation": "gt", "question": "exhibits से उन पंक्तियों को प्रदर्शित करें जहाँ salary 0 से ज्यादा है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE salary > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE salary > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bfb3f8261ddb2fc2bf28_hi", "language": "hi", "backend": "mysql", "operation": "bottom", "question": "price के बढ़ते क्रम में specimens की 3 सबसे कम पंक्तियां, सबसे कम से शुरू होकर, लें।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens ORDER BY price ASC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens ORDER BY price ASC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dc59f00f50b6c8f7bc16_hi", "language": "hi", "backend": "mysql", "operation": "between", "question": "exhibits से वे सभी आइटम चुनें जहाँ amount कॉलम का मान 20 और 520 के बीच है, समावेशी।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM exhibits WHERE amount BETWEEN 20 AND 520;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "'sql'"}}
{"id": "c01aaa64706cccb8ac54_hi", "language": "hi", "backend": "mysql", "operation": "between", "question": "specimens से वे सभी आइटम चुनें जहाँ balance कॉलम का मान 0 और 50 के बीच है, समावेशी।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE balance BETWEEN 0 AND 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "'sql'"}}
{"id": "804d318fe46a0e664840_hi", "language": "hi", "backend": "mysql", "operation": "sort_asc", "question": "reservations से सभी डेटा quantity के बढ़ते हुए क्रम में लें और दिखाएं।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY quantity ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY quantity ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4e9184b509f43a710021_hi", "language": "hi", "backend": "mysql", "operation": "sort_desc", "question": "reservations की पूरी सारणी balance के उच्चतम से निम्नतम क्रम में दिखाएँ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations ORDER BY balance DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY balance DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2e9e0c695ac50e3982d7_hi", "language": "hi", "backend": "mysql", "operation": "month", "question": "मुझे exhibits से वे डेटा पंक्तियां दें जहाँ order_date का महीना 12 है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE MONTH(order_date) = 12;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE MONTH(order_date) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b10e3bcce6871b79df41_hi", "language": "hi", "backend": "mysql", "operation": "sort_asc", "question": "reservations से सभी डेटा price के बढ़ते हुए क्रम में लें और दिखाएं।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations ORDER BY price ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d1f372f544fa10f6df1b_hi", "language": "hi", "backend": "supabase", "operation": "join", "question": "कृपया reservations.group_id को reservations_groups.id से जोड़ें और reservations.title तथा reservations_groups.label प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT reservations.title, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT reservations.title, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5f0b5d842746b96c126b_hi", "language": "hi", "backend": "mysql", "operation": "between", "question": "exhibits से वे सभी आइटम चुनें जहाँ price कॉलम का मान 1000 और 1100 के बीच है, समावेशी।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price BETWEEN 1000 AND 1100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "'sql'"}}
{"id": "8f9ab277b699f2cbe988_hi", "language": "hi", "backend": "mysql", "operation": "sort_desc", "question": "reservations की पूरी सारणी score के उच्चतम से निम्नतम क्रम में दिखाएँ।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0d94a075cd7e5759bdfa_hi", "language": "hi", "backend": "mysql", "operation": "between", "question": "exhibits से वे सभी आइटम चुनें जहाँ rating कॉलम का मान 10 और 110 के बीच है, समावेशी।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE rating BETWEEN 10 AND 110;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "'query'"}}
{"id": "bd760f1ca7aff546e852_hi", "language": "hi", "backend": "mysql", "operation": "bottom", "question": "balance के बढ़ते क्रम में exhibits की 5 सबसे कम पंक्तियां, सबसे कम से शुरू होकर, लें।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits ORDER BY balance ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY balance ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "139a72e10281049d1fed_hi", "language": "hi", "backend": "mysql", "operation": "join", "question": "कृपया specimens.group_id को specimens_groups.id से जोड़ें और specimens.name तथा specimens_groups.label प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT specimens.name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT specimens.name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4e4b83d8af9eb4c5efc8_hi", "language": "hi", "backend": "mysql", "operation": "bottom", "question": "amount के बढ़ते क्रम में inspections की 5 सबसे कम पंक्तियां, सबसे कम से शुरू होकर, लें।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "523a83d6ca07c1a1334a_hi", "language": "hi", "backend": "supabase", "operation": "missing_value", "question": "department के आधार पर exhibits पर एक फ़िल्टर निष्पादित करें, लेकिन मैंने कोई मान निर्दिष्ट नहीं किया है।", "expected": {"action": "clarify", "question": "department का कौन सा मान चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"department का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "5d05a2e11d5abab8423e_hi", "language": "hi", "backend": "supabase", "operation": "missing_value", "question": "region के आधार पर inspections पर एक फ़िल्टर निष्पादित करें, लेकिन मैंने कोई मान निर्दिष्ट नहीं किया है।", "expected": {"action": "clarify", "question": "region का कौन सा मान चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"region का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "fd9cafbb7e6e0ce69d06_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "join", "question": "output reservations.item_name and reservations_groups.label by joining reservations.group_id to reservations_groups.id", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8a2e6d77aa1c3a517ecb_hi", "language": "hi", "backend": "mysql", "operation": "having", "question": "कृपया exhibits में 100 से अधिक पंक्तियों वाले category समूहों और उनकी गणना को प्रदर्शित करें।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT category, COUNT(*) FROM exhibits GROUP BY category HAVING COUNT(*) > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM exhibits GROUP BY category HAVING COUNT(*) > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fd9cafbb7e6e0ce69d06_en", "language": "en", "backend": "mysql", "operation": "join", "question": "Extract reservations.item_name and reservations_groups.label by joining reservations.group_id to reservations_groups.id.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "844b55044ac74adf4493_hi", "language": "hi", "backend": "supabase", "operation": "join", "question": "कृपया reservations.group_id को reservations_groups.id से जोड़ें और reservations.item_name तथा reservations_groups.label प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9956c110975a38c3d293_hi", "language": "hi", "backend": "mysql", "operation": "top", "question": "शीर्ष 3 पंक्तियाँ reservations से प्राप्त करें, cost के आधार पर उच्चतम पहले।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations ORDER BY cost DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY cost DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "00c24a7cc97ee9debf36_hi", "language": "hi", "backend": "mysql", "operation": "contains", "question": "exhibits से वे पंक्तियाँ दिखाएं जिनमें title pending के समान है, मामला अंतर नजरअंदाज करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM exhibits WHERE LOWER(title) LIKE LOWER('%pending%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE LOWER(title) LIKE LOWER('%pending%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fd9cafbb7e6e0ce69d06_hinglish", "language": "hinglish", "backend": "mysql", "operation": "join", "question": "Kripya reservations.group_id ko reservations_groups.id se join karke reservations.item_name aur reservations_groups.label retrieve karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "307d5b7bd93949f7381a_hi", "language": "hi", "backend": "mysql", "operation": "join", "question": "कृपया specimens.group_id को specimens_groups.id से जोड़ें और specimens.customer_name तथा specimens_groups.label प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "53220f60789b4f5e4046_hi", "language": "hi", "backend": "mysql", "operation": "join", "question": "कृपया specimens.group_id को specimens_groups.id से जोड़ें और specimens.customer_name तथा specimens_groups.label प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f97582c94f2100eb1007_hi", "language": "hi", "backend": "mysql", "operation": "having", "question": "कृपया exhibits में 500 से अधिक पंक्तियों वाले department समूहों और उनकी गणना को प्रदर्शित करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a7675c74645d63d69aac_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "join_filter", "question": "show me reservations.description for reservations entries where reservations.group_id links to reservations_groups.id and reservations_groups.label is pending", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a7675c74645d63d69aac_hinglish", "language": "hinglish", "backend": "mysql", "operation": "join_filter", "question": "Please reservations.description dikhao jahan reservations reservations.group_id ke through reservations_groups se juda hai aur reservations_groups.label pending hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1b32b024d0c001fd9ae4_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "join_filter", "question": "show me reservations.description for reservations entries where reservations.group_id links to reservations_groups.id and reservations_groups.label is O'Reilly", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1b32b024d0c001fd9ae4_hinglish", "language": "hinglish", "backend": "mysql", "operation": "join_filter", "question": "Please reservations.description dikhao jahan reservations reservations.group_id ke through reservations_groups se juda hai aur reservations_groups.label O'Reilly hai.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a30eeaf286c79c95a1cd_hi", "language": "hi", "backend": "supabase", "operation": "join_filter", "question": "कृपया inspections.item_name दिखाएँ जहाँ inspections inspections.group_id के माध्यम से inspections_groups से जुड़ा है और inspections_groups.label O'Reilly है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT inspections.item_name FROM inspections JOIN inspections_groups ON inspections.group_id = inspections_groups.id WHERE inspections_groups.label = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT inspections.item_name FROM inspections JOIN inspections_groups ON inspections.group_id = inspections_groups.id WHERE inspections_groups.label = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a7675c74645d63d69aac_en", "language": "en", "backend": "mysql", "operation": "join_filter", "question": "Give me reservations.description for reservations entries where reservations.group_id points to reservations_groups.id and reservations_groups.label is pending.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "30a7ec8a99d4ee6b6d0e_hi", "language": "hi", "backend": "mysql", "operation": "date_after", "question": "inspections में से वे पंक्तियाँ चुनें जिनके order_date में 2026-12-15 या उससे बाद की तारीख है।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM inspections WHERE order_date >= '2026-12-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE order_date >= '2026-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1b32b024d0c001fd9ae4_en", "language": "en", "backend": "mysql", "operation": "join_filter", "question": "Give me reservations.description for reservations entries where reservations.group_id points to reservations_groups.id and reservations_groups.label is O'Reilly.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2cf634192448e68935c1_hi", "language": "hi", "backend": "mysql", "operation": "final", "question": "डेटाबेस टूल ने 5 पंक्तियां वापस कीं। कितनी पंक्तियां लौटीं?", "expected": {"action": "answer", "text": "टूल ने 5 पंक्तियाँ लौटाईं।"}, "output": "{\"action\":\"answer\",\"text\":\"टूल ने 5 पंक्तियाँ लौटाईं।\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "a59c6e6bcec70a1667e5_hi", "language": "hi", "backend": "mysql", "operation": "final", "question": "डेटाबेस टूल ने 35 पंक्तियां वापस कीं। कितनी पंक्तियां लौटीं?", "expected": {"action": "answer", "text": "टूल ने 35 पंक्तियाँ लौटाईं।"}, "output": "{\"action\":\"answer\",\"text\":\"टूल ने 35 पंक्तियाँ लौटाईं।\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "01f4f9527d44f09ab6b6_hi", "language": "hi", "backend": "mysql", "operation": "having", "question": "कृपया reservations में 200 से अधिक पंक्तियों वाले kind समूहों और उनकी गणना को प्रदर्शित करें।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM reservations GROUP BY kind HAVING COUNT(*) > 200;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM reservations GROUP BY kind HAVING COUNT(*) > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "55c82b78d80d85926577_hi", "language": "hi", "backend": "mysql", "operation": "date_after", "question": "exhibits में से वे पंक्तियाँ चुनें जिनके start_date में 2026-09-15 या उससे बाद की तारीख है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE start_date >= '2026-09-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE start_date >= '2026-09-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4a37ab647af0a5a5cb30_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "join_filter", "question": "show me reservations.display_name for reservations entries where reservations.group_id links to reservations_groups.id and reservations_groups.label is Electronics", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT reservations.display_name FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT reservations.display_name FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4a37ab647af0a5a5cb30_hinglish", "language": "hinglish", "backend": "mysql", "operation": "join_filter", "question": "Please reservations.display_name dikhao jahan reservations reservations.group_id ke through reservations_groups se juda hai aur reservations_groups.label Electronics hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT reservations.display_name FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT reservations.display_name FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4a37ab647af0a5a5cb30_en", "language": "en", "backend": "mysql", "operation": "join_filter", "question": "Give me reservations.display_name for reservations entries where reservations.group_id points to reservations_groups.id and reservations_groups.label is Electronics.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT reservations.display_name FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT reservations.display_name FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "63fa94106f0cf26f7da2_hi", "language": "hi", "backend": "mysql", "operation": "join_filter", "question": "कृपया specimens.title दिखाएँ जहाँ specimens specimens.group_id के माध्यम से specimens_groups से जुड़ा है और specimens_groups.label North है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT specimens.title FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id WHERE specimens_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.title FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id WHERE specimens_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f55358178d003de7ba56_hi", "language": "hi", "backend": "mysql", "operation": "final", "question": "डेटाबेस टूल ने 133 पंक्तियां वापस कीं। कितनी पंक्तियां लौटीं?", "expected": {"action": "answer", "text": "टूल ने 133 पंक्तियाँ लौटाईं।"}, "output": "{\"action\":\"answer\",\"text\":\"टूल ने 133 पंक्तियाँ लौटाईं।\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "40ffef08b12e4a3276e3_hi", "language": "hi", "backend": "mysql", "operation": "final", "question": "डेटाबेस टूल ने 116 पंक्तियां वापस कीं। कितनी पंक्तियां लौटीं?", "expected": {"action": "answer", "text": "टूल ने 116 पंक्तियाँ लौटाईं।"}, "output": "{\"action\":\"answer\",\"text\":\"टूल ने 116 पंक्तियाँ लौटाईं।\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "5f07a4c7c387316664f1_hi", "language": "hi", "backend": "mysql", "operation": "missing_value", "question": "department के आधार पर exhibits पर एक फ़िल्टर निष्पादित करें, लेकिन मैंने कोई मान निर्दिष्ट नहीं किया है।", "expected": {"action": "clarify", "question": "department का कौन सा मान चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"department का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "3fdbd39c598af9b9ea10_hi", "language": "hi", "backend": "mysql", "operation": "missing_value", "question": "status के आधार पर reservations पर एक फ़िल्टर निष्पादित करें, लेकिन मैंने कोई मान निर्दिष्ट नहीं किया है।", "expected": {"action": "clarify", "question": "status का कौन सा मान चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"status का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "fd9cafbb7e6e0ce69d06_hi", "language": "hi", "backend": "mysql", "operation": "join", "question": "कृपया reservations.group_id को reservations_groups.id से जोड़ें और reservations.item_name तथा reservations_groups.label प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a7675c74645d63d69aac_hi", "language": "hi", "backend": "mysql", "operation": "join_filter", "question": "कृपया reservations.description दिखाएँ जहाँ reservations reservations.group_id के माध्यम से reservations_groups से जुड़ा है और reservations_groups.label pending है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1b32b024d0c001fd9ae4_hi", "language": "hi", "backend": "mysql", "operation": "join_filter", "question": "कृपया reservations.description दिखाएँ जहाँ reservations reservations.group_id के माध्यम से reservations_groups से जुड़ा है और reservations_groups.label O'Reilly है।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4a37ab647af0a5a5cb30_hi", "language": "hi", "backend": "mysql", "operation": "join_filter", "question": "कृपया reservations.display_name दिखाएँ जहाँ reservations reservations.group_id के माध्यम से reservations_groups से जुड़ा है और reservations_groups.label Electronics है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT reservations.display_name FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT reservations.display_name FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "88b478ce80eed61bdfb4_hinglish", "language": "hinglish", "backend": "supabase", "operation": "weather", "question": "Jaipur ka mausam celsius mein check karo.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6ba167225d0a0648e865_hinglish", "language": "hinglish", "backend": "supabase", "operation": "weather", "question": "Lucknow ka mausam celsius mein check karo.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Lucknow", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "d84c7c968e50589a3321_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "search", "question": "look up API authentication in the doc", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "API authentication"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"API authentication\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "d84c7c968e50589a3321_hinglish", "language": "hinglish", "backend": "supabase", "operation": "search", "question": "Documentation ko search karke API authentication nikalo.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "API authentication"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"API authentication\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "88b478ce80eed61bdfb4_en", "language": "en", "backend": "supabase", "operation": "weather", "question": "Access the weather data for Jaipur in celsius.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "88b478ce80eed61bdfb4_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "weather", "question": "Get the weather status for Jaipur using celsius.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6ba167225d0a0648e865_en", "language": "en", "backend": "supabase", "operation": "weather", "question": "Access the weather data for Lucknow in celsius.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Lucknow", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6ba167225d0a0648e865_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "weather", "question": "Get the weather status for Lucknow using celsius.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Lucknow", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "d84c7c968e50589a3321_en", "language": "en", "backend": "supabase", "operation": "search", "question": "Search through the documentation to find API authentication.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "API authentication"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"API authentication\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "266ea8aafdf18d7c9212_en", "language": "en", "backend": "supabase", "operation": "ticket", "question": "Access the data for ticket TKT-6369.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6369"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6369\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "cca31e425efaaadf4c97_en", "language": "en", "backend": "supabase", "operation": "ticket", "question": "Access the data for ticket TKT-9184.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-9184"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9184\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "266ea8aafdf18d7c9212_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ticket", "question": "i need to look at support ticket TKT-6369", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6369"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6369\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "266ea8aafdf18d7c9212_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ticket", "question": "support ticket TKT-6369 ki details show karo.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6369"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6369\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "cca31e425efaaadf4c97_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ticket", "question": "i need to look at support ticket TKT-9184", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-9184"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9184\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "cca31e425efaaadf4c97_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ticket", "question": "support ticket TKT-9184 ki details show karo.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-9184"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9184\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "b6a1ad15475eea3c8bfb_hinglish", "language": "hinglish", "backend": "supabase", "operation": "read_file", "question": "/reports/sales.csv par file padh kar data lao.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/reports/sales.csv"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "88b478ce80eed61bdfb4_hi", "language": "hi", "backend": "supabase", "operation": "weather", "question": "Jaipur का मौसम celsius में पता करें।", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6ba167225d0a0648e865_hi", "language": "hi", "backend": "supabase", "operation": "weather", "question": "Lucknow का मौसम celsius में पता करें।", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Lucknow", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "b6a1ad15475eea3c8bfb_en", "language": "en", "backend": "supabase", "operation": "read_file", "question": "Retrieve the data by reading the file at /reports/sales.csv.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/reports/sales.csv"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "b6a1ad15475eea3c8bfb_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "read_file", "question": "Get the data from the file at /reports/sales.csv by reading it.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/reports/sales.csv"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "d84c7c968e50589a3321_hi", "language": "hi", "backend": "supabase", "operation": "search", "question": "दस्तावेज़ों में API authentication का उल्लेख खोजें।", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "API authentication"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"API authentication\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "266ea8aafdf18d7c9212_hi", "language": "hi", "backend": "supabase", "operation": "ticket", "question": "सपोर्ट टिकट TKT-6369 की जानकारी दिखाई दीजिए।", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6369"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6369\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "4e42bbd40467b2b8c27f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "search", "question": "look up SQL joins in the doc", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "SQL joins"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "4e42bbd40467b2b8c27f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "search", "question": "Documentation ko search karke SQL joins nikalo.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "SQL joins"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "cca31e425efaaadf4c97_hi", "language": "hi", "backend": "supabase", "operation": "ticket", "question": "सपोर्ट टिकट TKT-9184 की जानकारी दिखाई दीजिए।", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-9184"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9184\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "635a24ec4bc42b67226a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "weather", "question": "Jaipur ka mausam celsius mein check karo.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "4e42bbd40467b2b8c27f_en", "language": "en", "backend": "supabase", "operation": "search", "question": "Search through the documentation to find SQL joins.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "SQL joins"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "b6a1ad15475eea3c8bfb_hi", "language": "hi", "backend": "supabase", "operation": "read_file", "question": "/reports/sales.csv पर स्थित फ़ाइल को पढ़कर डेटा प्राप्त करें।", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/reports/sales.csv"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "635a24ec4bc42b67226a_en", "language": "en", "backend": "supabase", "operation": "weather", "question": "Access the weather data for Jaipur in celsius.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "635a24ec4bc42b67226a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "weather", "question": "Get the weather status for Jaipur using celsius.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "8a696e3c3522dd69d5ac_en", "language": "en", "backend": "supabase", "operation": "ticket", "question": "Access the data for ticket TKT-4486.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-4486"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-4486\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "3dbbf63e641d9abf49db_en", "language": "en", "backend": "supabase", "operation": "ticket", "question": "Access the data for ticket TKT-9979.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-9979"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9979\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "8a696e3c3522dd69d5ac_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ticket", "question": "i need to look at support ticket TKT-4486", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-4486"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-4486\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "8a696e3c3522dd69d5ac_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ticket", "question": "support ticket TKT-4486 ki details show karo.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-4486"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-4486\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "3dbbf63e641d9abf49db_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ticket", "question": "i need to look at support ticket TKT-9979", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-9979"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9979\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "3dbbf63e641d9abf49db_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ticket", "question": "support ticket TKT-9979 ki details show karo.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-9979"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9979\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "89c9ecdcb7242ef71d8b_en", "language": "en", "backend": "supabase", "operation": "ticket", "question": "Access the data for ticket TKT-5087.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-5087"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-5087\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "89c9ecdcb7242ef71d8b_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ticket", "question": "i need to look at support ticket TKT-5087", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-5087"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-5087\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "89c9ecdcb7242ef71d8b_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ticket", "question": "support ticket TKT-5087 ki details show karo.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-5087"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-5087\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "f73e21e0eb10d6617995_hinglish", "language": "hinglish", "backend": "supabase", "operation": "read_file", "question": "/config/app.json par file padh kar data lao.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "635a24ec4bc42b67226a_hi", "language": "hi", "backend": "supabase", "operation": "weather", "question": "Jaipur का मौसम celsius में पता करें।", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "0161f123d136665478c9_hinglish", "language": "hinglish", "backend": "supabase", "operation": "read_file", "question": "/notes/setup.txt par file padh kar data lao.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "f73e21e0eb10d6617995_en", "language": "en", "backend": "supabase", "operation": "read_file", "question": "Retrieve the data by reading the file at /config/app.json.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "0161f123d136665478c9_en", "language": "en", "backend": "supabase", "operation": "read_file", "question": "Retrieve the data by reading the file at /notes/setup.txt.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "f73e21e0eb10d6617995_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "read_file", "question": "Get the data from the file at /config/app.json by reading it.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "0161f123d136665478c9_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "read_file", "question": "Get the data from the file at /notes/setup.txt by reading it.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "4e42bbd40467b2b8c27f_hi", "language": "hi", "backend": "supabase", "operation": "search", "question": "दस्तावेज़ों में SQL joins का उल्लेख खोजें।", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "SQL joins"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "531ed6009fed93af93ef_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "search", "question": "look up indexes in the doc", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "indexes"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false, "error": "Unknown tool"}}
{"id": "531ed6009fed93af93ef_hinglish", "language": "hinglish", "backend": "mysql", "operation": "search", "question": "Documentation ko search karke indexes nikalo.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "indexes"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false, "error": "Unknown tool"}}
{"id": "8c8ba16d1f868e7d81a9_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "search", "question": "look up indexes in the doc", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "indexes"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "8c8ba16d1f868e7d81a9_hinglish", "language": "hinglish", "backend": "mysql", "operation": "search", "question": "Documentation ko search karke indexes nikalo.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "indexes"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "252c6053295d69a93c94_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "search", "question": "look up database backups in the doc", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "252c6053295d69a93c94_hinglish", "language": "hinglish", "backend": "mysql", "operation": "search", "question": "Documentation ko search karke database backups nikalo.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "531ed6009fed93af93ef_en", "language": "en", "backend": "mysql", "operation": "search", "question": "Search through the documentation to find indexes.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "indexes"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false, "error": "Unknown tool"}}
{"id": "8c8ba16d1f868e7d81a9_en", "language": "en", "backend": "mysql", "operation": "search", "question": "Search through the documentation to find indexes.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "indexes"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "252c6053295d69a93c94_en", "language": "en", "backend": "mysql", "operation": "search", "question": "Search through the documentation to find database backups.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "5ebc52b3d20bde2076b3_en", "language": "en", "backend": "mysql", "operation": "ticket", "question": "Access the data for ticket TKT-6124.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6124"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "754691bd1eea25d41ccf_hinglish", "language": "hinglish", "backend": "mysql", "operation": "weather", "question": "Delhi ka mausam fahrenheit mein check karo.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6600e72cfc86012212d0_hinglish", "language": "hinglish", "backend": "mysql", "operation": "weather", "question": "Delhi ka mausam fahrenheit mein check karo.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "b5839920055891fb68c8_hinglish", "language": "hinglish", "backend": "mysql", "operation": "weather", "question": "Lucknow ka mausam celsius mein check karo.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Lucknow", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "5ebc52b3d20bde2076b3_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "ticket", "question": "i need to look at support ticket TKT-6124", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6124"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "5ebc52b3d20bde2076b3_hinglish", "language": "hinglish", "backend": "mysql", "operation": "ticket", "question": "support ticket TKT-6124 ki details show karo.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6124"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "754691bd1eea25d41ccf_en", "language": "en", "backend": "mysql", "operation": "weather", "question": "Access the weather data for Delhi in fahrenheit.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "754691bd1eea25d41ccf_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "weather", "question": "Get the weather status for Delhi using fahrenheit.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6600e72cfc86012212d0_en", "language": "en", "backend": "mysql", "operation": "weather", "question": "Access the weather data for Delhi in fahrenheit.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6600e72cfc86012212d0_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "weather", "question": "Get the weather status for Delhi using fahrenheit.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "b5839920055891fb68c8_en", "language": "en", "backend": "mysql", "operation": "weather", "question": "Access the weather data for Lucknow in celsius.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Lucknow", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "b5839920055891fb68c8_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "weather", "question": "Get the weather status for Lucknow using celsius.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Lucknow", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "adad513dde5be462c95d_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "search", "question": "look up row level security in the doc", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "row level security"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "adad513dde5be462c95d_hinglish", "language": "hinglish", "backend": "mysql", "operation": "search", "question": "Documentation ko search karke row level security nikalo.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "row level security"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "adad513dde5be462c95d_en", "language": "en", "backend": "mysql", "operation": "search", "question": "Search through the documentation to find row level security.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "row level security"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6f3055c580b33340c854_hinglish", "language": "hinglish", "backend": "mysql", "operation": "read_file", "question": "/notes/setup.txt par file padh kar data lao.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "ae382a2348421f540f9e_hinglish", "language": "hinglish", "backend": "mysql", "operation": "read_file", "question": "/config/app.json par file padh kar data lao.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "754691bd1eea25d41ccf_hi", "language": "hi", "backend": "mysql", "operation": "weather", "question": "Delhi का मौसम fahrenheit में पता करें।", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "8a696e3c3522dd69d5ac_hi", "language": "hi", "backend": "supabase", "operation": "ticket", "question": "सपोर्ट टिकट TKT-4486 की जानकारी दिखाई दीजिए।", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-4486"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-4486\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6600e72cfc86012212d0_hi", "language": "hi", "backend": "mysql", "operation": "weather", "question": "Delhi का मौसम fahrenheit में पता करें।", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "b5839920055891fb68c8_hi", "language": "hi", "backend": "mysql", "operation": "weather", "question": "Lucknow का मौसम celsius में पता करें।", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Lucknow", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6f3055c580b33340c854_en", "language": "en", "backend": "mysql", "operation": "read_file", "question": "Retrieve the data by reading the file at /notes/setup.txt.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "13d062ce5950bde16f8f_hinglish", "language": "hinglish", "backend": "mysql", "operation": "read_file", "question": "/docs/README.md par file padh kar data lao.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/docs/README.md"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/docs/README.md\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "3dbbf63e641d9abf49db_hi", "language": "hi", "backend": "supabase", "operation": "ticket", "question": "सपोर्ट टिकट TKT-9979 की जानकारी दिखाई दीजिए।", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-9979"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9979\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "ae382a2348421f540f9e_en", "language": "en", "backend": "mysql", "operation": "read_file", "question": "Retrieve the data by reading the file at /config/app.json.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "89c9ecdcb7242ef71d8b_hi", "language": "hi", "backend": "supabase", "operation": "ticket", "question": "सपोर्ट टिकट TKT-5087 की जानकारी दिखाई दीजिए।", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-5087"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-5087\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "f73e21e0eb10d6617995_hi", "language": "hi", "backend": "supabase", "operation": "read_file", "question": "/config/app.json पर स्थित फ़ाइल को पढ़कर डेटा प्राप्त करें।", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6f3055c580b33340c854_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "read_file", "question": "Get the data from the file at /notes/setup.txt by reading it.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "13d062ce5950bde16f8f_en", "language": "en", "backend": "mysql", "operation": "read_file", "question": "Retrieve the data by reading the file at /docs/README.md.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/docs/README.md"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/docs/README.md\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "ae382a2348421f540f9e_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "read_file", "question": "Get the data from the file at /config/app.json by reading it.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "0161f123d136665478c9_hi", "language": "hi", "backend": "supabase", "operation": "read_file", "question": "/notes/setup.txt पर स्थित फ़ाइल को पढ़कर डेटा प्राप्त करें।", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "13d062ce5950bde16f8f_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "read_file", "question": "Get the data from the file at /docs/README.md by reading it.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/docs/README.md"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/docs/README.md\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "531ed6009fed93af93ef_hi", "language": "hi", "backend": "mysql", "operation": "search", "question": "दस्तावेज़ों में indexes का उल्लेख खोजें।", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "indexes"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false, "error": "Unknown tool"}}
{"id": "8c8ba16d1f868e7d81a9_hi", "language": "hi", "backend": "mysql", "operation": "search", "question": "दस्तावेज़ों में indexes का उल्लेख खोजें।", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "indexes"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "252c6053295d69a93c94_hi", "language": "hi", "backend": "mysql", "operation": "search", "question": "दस्तावेज़ों में database backups का उल्लेख खोजें।", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "adad513dde5be462c95d_hi", "language": "hi", "backend": "mysql", "operation": "search", "question": "दस्तावेज़ों में row level security का उल्लेख खोजें।", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "row level security"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "5ebc52b3d20bde2076b3_hi", "language": "hi", "backend": "mysql", "operation": "ticket", "question": "सपोर्ट टिकट TKT-6124 की जानकारी दिखाई दीजिए।", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6124"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6f3055c580b33340c854_hi", "language": "hi", "backend": "mysql", "operation": "read_file", "question": "/notes/setup.txt पर स्थित फ़ाइल को पढ़कर डेटा प्राप्त करें।", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "ae382a2348421f540f9e_hi", "language": "hi", "backend": "mysql", "operation": "read_file", "question": "/config/app.json पर स्थित फ़ाइल को पढ़कर डेटा प्राप्त करें।", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "13d062ce5950bde16f8f_hi", "language": "hi", "backend": "mysql", "operation": "read_file", "question": "/docs/README.md पर स्थित फ़ाइल को पढ़कर डेटा प्राप्त करें।", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/docs/README.md"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/docs/README.md\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}