{"id": "2e299c717e66247d2e4b_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "missing_schema", "question": "I didn't provide schema. Please find the available tables before writing sql for repairs.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "2e299c717e66247d2e4b_en", "language": "en", "backend": "supabase", "operation": "missing_schema", "question": "Please determine the available tables before writing SQL for repairs, since I did not provide a database schema.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "2e299c717e66247d2e4b_hinglish", "language": "hinglish", "backend": "supabase", "operation": "missing_schema", "question": "Maine database schema provide nahi kiya hai, isliye repairs ke liye SQL likhne se pehle available tables find karo.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a509f9b46620c564c0cf_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "missing_schema", "question": "I didn't provide schema. Please find the available tables before writing sql for repairs.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_repairs", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_repairs\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a509f9b46620c564c0cf_en", "language": "en", "backend": "supabase", "operation": "missing_schema", "question": "Please determine the available tables before writing SQL for repairs, since I did not provide a database schema.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_repairs", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_repairs\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a509f9b46620c564c0cf_hinglish", "language": "hinglish", "backend": "supabase", "operation": "missing_schema", "question": "Maine database schema provide nahi kiya hai, isliye repairs ke liye SQL likhne se pehle available tables find karo.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_repairs", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_repairs\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "07beb228a10fcdc90b00_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "missing_schema", "question": "I didn't provide schema. Please find the available tables before writing sql for enrollments.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6230fff878aa525542cd_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "missing_schema", "question": "I didn't provide schema. Please find the available tables before writing sql for enrollments.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "94b5cc87d7b229fd2427_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "missing_schema", "question": "I didn't provide schema. Please find the available tables before writing sql for deliveries.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "07beb228a10fcdc90b00_en", "language": "en", "backend": "mysql", "operation": "missing_schema", "question": "Please determine the available tables before writing SQL for enrollments, since I did not provide a database schema.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6230fff878aa525542cd_en", "language": "en", "backend": "mysql", "operation": "missing_schema", "question": "Please determine the available tables before writing SQL for enrollments, since I did not provide a database schema.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "aca1abed450f2955c4ab_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "missing_schema", "question": "I didn't provide schema. Please find the available tables before writing sql for deliveries.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "07beb228a10fcdc90b00_hinglish", "language": "hinglish", "backend": "mysql", "operation": "missing_schema", "question": "Maine database schema provide nahi kiya hai, isliye enrollments ke liye SQL likhne se pehle available tables find karo.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6230fff878aa525542cd_hinglish", "language": "hinglish", "backend": "mysql", "operation": "missing_schema", "question": "Maine database schema provide nahi kiya hai, isliye enrollments ke liye SQL likhne se pehle available tables find karo.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "c4df2d6a92784c925f92_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "all", "question": "pull all rows from rentals", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "94b5cc87d7b229fd2427_en", "language": "en", "backend": "mysql", "operation": "missing_schema", "question": "Please determine the available tables before writing SQL for deliveries, since I did not provide a database schema.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "c4df2d6a92784c925f92_hinglish", "language": "hinglish", "backend": "supabase", "operation": "all", "question": "rentals se saare rows le aao.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d1d96cfbe28ba036d11d_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "write", "question": "Clear rentals completely.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "94b5cc87d7b229fd2427_hinglish", "language": "hinglish", "backend": "mysql", "operation": "missing_schema", "question": "Maine database schema provide nahi kiya hai, isliye deliveries ke liye SQL likhne se pehle available tables find karo.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "c4df2d6a92784c925f92_en", "language": "en", "backend": "supabase", "operation": "all", "question": "Bring up all rows from rentals.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "aca1abed450f2955c4ab_en", "language": "en", "backend": "mysql", "operation": "missing_schema", "question": "Please determine the available tables before writing SQL for deliveries, since I did not provide a database schema.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "04cdd3f064826214f853_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "distinct", "question": "find the unique status in enrollments", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT DISTINCT status FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT status FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4658c817cf80ab71fd79_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "max", "question": "get the maximum rating from rentals please", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT MAX(rating) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(rating) FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a7d907a9a18c081961a8_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "distinct", "question": "find the unique status in enrollments", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT DISTINCT status FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT status FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f39ec09a8da50e72bdd5_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "min", "question": "Show me minimum price in enrollments.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT MIN(price) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(price) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "aca1abed450f2955c4ab_hinglish", "language": "hinglish", "backend": "mysql", "operation": "missing_schema", "question": "Maine database schema provide nahi kiya hai, isliye deliveries ke liye SQL likhne se pehle available tables find karo.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "4658c817cf80ab71fd79_en", "language": "en", "backend": "supabase", "operation": "max", "question": "Give me the highest rating for rentals.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT MAX(rating) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(rating) FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4658c817cf80ab71fd79_hinglish", "language": "hinglish", "backend": "supabase", "operation": "max", "question": "rentals mein rating ki maximum number kya hai?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT MAX(rating) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(rating) FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "cdc7d96784e2e9f3c94b_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "distinct", "question": "find the unique department in rentals", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT DISTINCT department FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5815b9336617abb93d33_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "avg", "question": "give me avg rating in enrollments", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT AVG(rating) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(rating) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f39ec09a8da50e72bdd5_en", "language": "en", "backend": "supabase", "operation": "min", "question": "Give me the minimum price from enrollments.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT MIN(price) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(price) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c23d51b5c96bb4b300bf_hinglish", "language": "hinglish", "backend": "supabase", "operation": "group_count", "question": "rentals me har status ke rows ka count do.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT status, COUNT(*) FROM rentals GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT status, COUNT(*) FROM rentals GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2e299c717e66247d2e4b_hi", "language": "hi", "backend": "supabase", "operation": "missing_schema", "question": "मेने डेटाबेस स्कीमा प्रदान नहीं की है, इसलिए repairs के लिए SQL लिखने से पहले उपलब्ध तालिकाओं को खोजें।", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "d1d96cfbe28ba036d11d_hinglish", "language": "hinglish", "backend": "supabase", "operation": "write", "question": "rentals se har ek row remove karo.", "expected": {"action": "clarify", "question": "Yeh connection read-only hai. Kya SELECT query chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"Yeh connection read-only hai. Kya SELECT query chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "da7c6370afcc35f790f8_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sum", "question": "please find the total quantity in rentals", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT SUM(quantity) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(quantity) FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "99c40dbbcc3778d58436_hinglish", "language": "hinglish", "backend": "supabase", "operation": "group_count", "question": "rentals me har status ke rows ka count do.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT status, COUNT(*) FROM rentals GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT status, COUNT(*) FROM rentals GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f39ec09a8da50e72bdd5_hinglish", "language": "hinglish", "backend": "supabase", "operation": "min", "question": "enrollments mein price ka minimum limit kya hai?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT MIN(price) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(price) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c23d51b5c96bb4b300bf_en", "language": "en", "backend": "supabase", "operation": "group_count", "question": "Sum up the rows for each status in rentals.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT status, COUNT(*) FROM rentals GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT status, COUNT(*) FROM rentals GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c23d51b5c96bb4b300bf_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "group_count", "question": "i need the total rows for each status in rentals", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT status, COUNT(*) FROM rentals GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT status, COUNT(*) FROM rentals GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d2ac91503b4c4b8c4bbf_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "max", "question": "get the maximum quantity from enrollments please", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT MAX(quantity) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(quantity) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7513979dd6549473de83_en", "language": "en", "backend": "supabase", "operation": "count_eq", "question": "How many instances in rentals have city as active?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM rentals WHERE city = 'active';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM rentals WHERE city = 'active';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "99c40dbbcc3778d58436_en", "language": "en", "backend": "supabase", "operation": "group_count", "question": "Sum up the rows for each status in rentals.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT status, COUNT(*) FROM rentals GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT status, SUM(balance) FROM rentals GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "99c40dbbcc3778d58436_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "group_count", "question": "i need the total rows for each status in rentals", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT status, COUNT(*) FROM rentals GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT status, COUNT(*) FROM rentals GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5815b9336617abb93d33_en", "language": "en", "backend": "supabase", "operation": "avg", "question": "What is the rating average in enrollments?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT AVG(rating) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(rating) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5815b9336617abb93d33_hinglish", "language": "hinglish", "backend": "supabase", "operation": "avg", "question": "enrollments me rating ka average kya hai?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT AVG(rating) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(rating) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8488c3423e53e10c7853_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "count", "question": "count the rows in repairs", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d2ac91503b4c4b8c4bbf_en", "language": "en", "backend": "supabase", "operation": "max", "question": "Give me the highest quantity for enrollments.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT MAX(quantity) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(quantity) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7513979dd6549473de83_hinglish", "language": "hinglish", "backend": "supabase", "operation": "count_eq", "question": "rentals mein city active wale data ki sankhya batayo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM rentals WHERE city = 'active';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM rentals WHERE city = 'active';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fed9927b6460e7bd91b2_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sql_error", "question": "You had a unknown column error in the last query. Check repairs schema before trying again.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_repairs", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_repairs\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a2ad743e655e7c8923dc_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project", "question": "bring up customer_name from rentals only", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT customer_name FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a2ad743e655e7c8923dc_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project", "question": "i want just customer_name from rentals ok", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT customer_name FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "da7c6370afcc35f790f8_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sum", "question": "rentals mein quantity ka total measure karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT SUM(quantity) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(quantity) FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7a57ce9685f252c72f44_en", "language": "en", "backend": "supabase", "operation": "lt", "question": "Present the data from rentals where quantity is below 3.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE quantity < 3;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE quantity < 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c4e9855d27b398c547a8_hinglish", "language": "hinglish", "backend": "supabase", "operation": "top", "question": "rentals mein balance ke highest values wali 1 rows display karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals ORDER BY balance DESC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals ORDER BY balance DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9bb785183a3dc77c2cfa_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "group_sum", "question": "show the total balance for each status in enrollments now", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT status, SUM(balance) FROM enrollments GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT status, SUM(balance) FROM enrollments GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d2ac91503b4c4b8c4bbf_hinglish", "language": "hinglish", "backend": "supabase", "operation": "max", "question": "enrollments mein quantity ki maximum number kya hai?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT MAX(quantity) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(quantity) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d1d96cfbe28ba036d11d_en", "language": "en", "backend": "supabase", "operation": "write", "question": "Blow away all rows from rentals.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}}
{"id": "36d48e67b84ab1dd10fa_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project", "question": "bring up display_name from enrollments only", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT display_name FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "36d48e67b84ab1dd10fa_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project", "question": "i want just display_name from enrollments ok", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT display_name FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7513979dd6549473de83_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "count_eq", "question": "help me count rows in rentals where city is active", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM rentals WHERE city = 'active';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM rentals WHERE city = 'active';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7a57ce9685f252c72f44_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lt", "question": "output the records from rentals where quantity is below 3 please.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE quantity < 3;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE quantity < 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c4e9855d27b398c547a8_en", "language": "en", "backend": "supabase", "operation": "top", "question": "Present the top 1 records from rentals by descending balance value.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals ORDER BY balance DESC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals ORDER BY balance DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c4e9855d27b398c547a8_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "top", "question": "Show the top 1 entries in rentals by balance desc order.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals ORDER BY balance DESC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals ORDER BY balance DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e971c6c1ded7274f9394_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sum", "question": "please find the total balance in deliveries", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT SUM(balance) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(balance) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "04cdd3f064826214f853_en", "language": "en", "backend": "supabase", "operation": "distinct", "question": "Pull out the distinct status values from the data in enrollments.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT DISTINCT status FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT status FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "04cdd3f064826214f853_hinglish", "language": "hinglish", "backend": "supabase", "operation": "distinct", "question": "enrollments mein maujood data se status ke distinct values pull karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT DISTINCT status FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT status FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4c485cb9781970855140_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sort_desc", "question": "show all enrollments rows ordered by quantity highest to lowest", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments ORDER BY quantity DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments ORDER BY quantity DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b6bd83761ee381e1df8c_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "max", "question": "get the maximum balance from repairs please", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT MAX(balance) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(balance) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "da7c6370afcc35f790f8_en", "language": "en", "backend": "supabase", "operation": "sum", "question": "Measure the total quantity present in rentals.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT SUM(quantity) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(quantity) FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a7d907a9a18c081961a8_en", "language": "en", "backend": "supabase", "operation": "distinct", "question": "Pull out the distinct status values from the data in enrollments.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT DISTINCT status FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT status FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a7d907a9a18c081961a8_hinglish", "language": "hinglish", "backend": "supabase", "operation": "distinct", "question": "enrollments mein maujood data se status ke distinct values pull karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT DISTINCT status FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT status FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d169f3070176e20c972f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ambiguous", "question": "Show the leading records in deliveries.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "7782f04c53bb2061153a_en", "language": "en", "backend": "supabase", "operation": "lt", "question": "Present the data from rentals where amount is below 1000.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE amount < 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE amount < 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c3c868696c82e387acd8_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gte", "question": "get rows from enrollments where balance is 0 or above", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE balance >= 0;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE balance >= 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7a57ce9685f252c72f44_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lt", "question": "rentals se un rows ko show karo jahan quantity 3 se kam hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE quantity < 3;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE quantity < 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "cdc7d96784e2e9f3c94b_hinglish", "language": "hinglish", "backend": "supabase", "operation": "distinct", "question": "rentals mein maujood data se department ke distinct values pull karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT DISTINCT department FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bb6e110cb30376d23598_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sort_desc", "question": "show all enrollments rows ordered by price highest to lowest", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments ORDER BY price DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments ORDER BY price DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dbb8cd8445dd19a458e2_en", "language": "en", "backend": "supabase", "operation": "eq", "question": "Query enrollments to show all rows where department is Pune.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE department = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE department = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dbb8cd8445dd19a458e2_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "eq", "question": "Show all rows in enrollments where department is Pune ok?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE department = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE department = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "327bced9ccce8adafb67_en", "language": "en", "backend": "supabase", "operation": "lt", "question": "Present the data from enrollments where price is below 20.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE price < 20;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE price < 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8e0d860cde6e650084c2_en", "language": "en", "backend": "supabase", "operation": "or", "question": "Select rows from enrollments where kind is Electronics or Mumbai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE kind IN ('Electronics', 'Mumbai');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE kind IN ('Electronics', 'Mumbai');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8e0d860cde6e650084c2_hinglish", "language": "hinglish", "backend": "supabase", "operation": "or", "question": "enrollments se kind Electronics ya Mumbai wali rows le lo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE kind IN ('Electronics', 'Mumbai');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE kind IN ('Electronics', 'Mumbai');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b6bd83761ee381e1df8c_en", "language": "en", "backend": "supabase", "operation": "max", "question": "Give me the highest balance for repairs.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT MAX(balance) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(balance) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a2ad743e655e7c8923dc_en", "language": "en", "backend": "supabase", "operation": "project", "question": "Let's look at customer_name in rentals specifically.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT customer_name FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM rentals WHERE department = 'Aetra';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "a81aea4a2431376817c8_en", "language": "en", "backend": "supabase", "operation": "or", "question": "Select rows from enrollments where kind is Mumbai or Mumbai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE kind IN ('Mumbai', 'Mumbai');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE kind IN ('Mumbai', 'Mumbai');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a81aea4a2431376817c8_hinglish", "language": "hinglish", "backend": "supabase", "operation": "or", "question": "enrollments se kind Mumbai ya Mumbai wali rows le lo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE kind IN ('Mumbai', 'Mumbai');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE kind IN ('Mumbai', 'Mumbai');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b8697f5a0689a023e448_hinglish", "language": "hinglish", "backend": "supabase", "operation": "month", "question": "enrollments mein created_at ka month 10 hone wali rows dikhao.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE EXTRACT(MONTH FROM created_at) = 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE EXTRACT(MONTH FROM created_at) = 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7782f04c53bb2061153a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lt", "question": "output the records from rentals where amount is below 1000 please.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE amount < 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE amount < 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c3c868696c82e387acd8_en", "language": "en", "backend": "supabase", "operation": "gte", "question": "Find records in enrollments where balance is not below 0.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE balance >= 0;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE balance >= 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f826d550b5dd9bf82144_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "avg", "question": "give me avg rating in repairs", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT AVG(rating) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(rating) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "cdc7d96784e2e9f3c94b_en", "language": "en", "backend": "supabase", "operation": "distinct", "question": "Pull out the distinct department values from the data in rentals.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT DISTINCT department FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9bb785183a3dc77c2cfa_en", "language": "en", "backend": "supabase", "operation": "group_sum", "question": "I want to see the total balance for each status in enrollments.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT status, SUM(balance) FROM enrollments GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT status, SUM(balance) FROM enrollments GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9bb785183a3dc77c2cfa_hinglish", "language": "hinglish", "backend": "supabase", "operation": "group_sum", "question": "Mujhe enrollments me har status ke liye balance ka total dikhana hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT status, SUM(balance) FROM enrollments GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT status, SUM(balance) FROM enrollments GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "39205adac55e98cd6749_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project_eq", "question": "rentals se display_name output karo jahan kind O'Reilly hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT display_name FROM rentals WHERE kind = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM rentals WHERE kind = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "068560b98b4de1eee24e_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "contains", "question": "Bring rows from rentals where description has active, no case distinction.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE LOWER(description) LIKE LOWER('%active%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE LOWER(description) LIKE LOWER('%active%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "327bced9ccce8adafb67_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lt", "question": "output the records from enrollments where price is below 20 please.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE price < 20;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE price < 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5ec7a0dca8e2804dbca9_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "not_null", "question": "Show the enrollments rows where item_name is not null.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE item_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE item_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8488c3423e53e10c7853_en", "language": "en", "backend": "supabase", "operation": "count", "question": "List the number of rows found in repairs.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "36d48e67b84ab1dd10fa_en", "language": "en", "backend": "supabase", "operation": "project", "question": "Let's look at display_name in enrollments specifically.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT display_name FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b6bd83761ee381e1df8c_hinglish", "language": "hinglish", "backend": "supabase", "operation": "max", "question": "repairs mein balance ki maximum number kya hai?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT MAX(balance) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(balance) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fed9927b6460e7bd91b2_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sql_error", "question": "Kyunki aakhri query unknown column ki wajah se fail hui, repairs schema inspect karne ke baad hi dobara try karo.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_repairs", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_repairs\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "15e384ce46e890333204_en", "language": "en", "backend": "supabase", "operation": "gt", "question": "Select the rows from enrollments where rating is greater than 500.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE rating > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE rating > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "15e384ce46e890333204_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gt", "question": "look up the rows in enrollments where rating is greater than 500", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE rating > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE rating > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b8697f5a0689a023e448_en", "language": "en", "backend": "supabase", "operation": "month", "question": "Provide rows from enrollments where created_at's month value is 10.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE EXTRACT(MONTH FROM created_at) = 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE EXTRACT(MONTH FROM created_at) = 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b8697f5a0689a023e448_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "month", "question": "give me the rows in enrollments where month of created_at is 10", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE EXTRACT(MONTH FROM created_at) = 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE EXTRACT(MONTH FROM created_at) = 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d169f3070176e20c972f_en", "language": "en", "backend": "supabase", "operation": "ambiguous", "question": "Reveal the leading records in deliveries.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "d169f3070176e20c972f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ambiguous", "question": "deliveries ke best records reveal karo.", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "7782f04c53bb2061153a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lt", "question": "rentals se un rows ko show karo jahan amount 1000 se kam hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE amount < 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE amount < 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c3c868696c82e387acd8_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gte", "question": "enrollments ki entries lao jahan balance 0 ya usse upar hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE balance >= 0;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE balance >= 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c4df2d6a92784c925f92_hi", "language": "hi", "backend": "supabase", "operation": "all", "question": "rentals से सभी पंक्तियाँ लाइए।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "39205adac55e98cd6749_en", "language": "en", "backend": "supabase", "operation": "project_eq", "question": "Output the display_name from rentals where kind is O'Reilly.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT display_name FROM rentals WHERE kind = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM rentals WHERE kind = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "39205adac55e98cd6749_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project_eq", "question": "show me the display_name in rentals where kind is O'Reilly", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT display_name FROM rentals WHERE kind = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM rentals WHERE kind = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ad6bf3f875b067153767_en", "language": "en", "backend": "supabase", "operation": "count_eq", "question": "How many instances in repairs have category as South?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM repairs WHERE category = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM repairs WHERE category = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5815b9336617abb93d33_hi", "language": "hi", "backend": "supabase", "operation": "avg", "question": "enrollments में rating का औसत क्या है?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT AVG(rating) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(rating) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8e0d860cde6e650084c2_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "or", "question": "Please show rows from enrollments where kind is either Electronics or Mumbai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE kind IN ('Electronics', 'Mumbai');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE kind IN ('Electronics', 'Mumbai');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e971c6c1ded7274f9394_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sum", "question": "deliveries mein balance ka total measure karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT SUM(balance) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(balance) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4c485cb9781970855140_en", "language": "en", "backend": "supabase", "operation": "sort_desc", "question": "Return all rows from enrollments with quantity in descending order.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments ORDER BY quantity DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments ORDER BY quantity DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4c485cb9781970855140_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sort_desc", "question": "enrollments ke saare rows quantity highest se lowest order mein return karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments ORDER BY quantity DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments ORDER BY quantity DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "15e384ce46e890333204_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gt", "question": "Mujhe enrollments se woh rows chahiye jahan rating 500 se zyada hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE rating > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE rating > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a81aea4a2431376817c8_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "or", "question": "Please show rows from enrollments where kind is either Mumbai or Mumbai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE kind IN ('Mumbai', 'Mumbai');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE kind IN ('Mumbai', 'Mumbai');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f826d550b5dd9bf82144_en", "language": "en", "backend": "supabase", "operation": "avg", "question": "What is the rating average in repairs?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT AVG(rating) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(rating) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f826d550b5dd9bf82144_hinglish", "language": "hinglish", "backend": "supabase", "operation": "avg", "question": "repairs me rating ka average kya hai?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT AVG(rating) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(rating) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "36117416028ec9713345_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lte", "question": "select deliveries rows where salary is not more than 10000", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE salary <= 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE salary <= 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bb6e110cb30376d23598_en", "language": "en", "backend": "supabase", "operation": "sort_desc", "question": "Return all rows from enrollments with price in descending order.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments ORDER BY price DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments ORDER BY price DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bb6e110cb30376d23598_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sort_desc", "question": "enrollments ke saare rows price highest se lowest order mein return karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments ORDER BY price DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments ORDER BY price DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dbb8cd8445dd19a458e2_hinglish", "language": "hinglish", "backend": "supabase", "operation": "eq", "question": "enrollments se query karo aur wo saari rows dikhao jahan department Pune hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE department = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE department = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "068560b98b4de1eee24e_hinglish", "language": "hinglish", "backend": "supabase", "operation": "contains", "question": "Bring karo rentals se un rows ko jinka description active hai, case difference ke bina.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE LOWER(description) LIKE LOWER('%active%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE LOWER(description) LIKE LOWER('%active%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "327bced9ccce8adafb67_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lt", "question": "enrollments se un rows ko show karo jahan price 20 se kam hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE price < 20;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE price < 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5ec7a0dca8e2804dbca9_en", "language": "en", "backend": "supabase", "operation": "not_null", "question": "Filter enrollments to show only rows where item_name is not null.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE item_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE item_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8488c3423e53e10c7853_hinglish", "language": "hinglish", "backend": "supabase", "operation": "count", "question": "repairs ki rows ka volume kya hai?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "de15f8143029f00970f6_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "bottom", "question": "give me the 1 lowest price data from deliveries ordered ascending", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries ORDER BY price ASC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries ORDER BY price ASC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2e35310942114f57ebb2_en", "language": "en", "backend": "supabase", "operation": "eq", "question": "Query repairs to show all rows where category is Books.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE category = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE category = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2e35310942114f57ebb2_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "eq", "question": "Show all rows in repairs where category is Books ok?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE category = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE category = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e971c6c1ded7274f9394_en", "language": "en", "backend": "supabase", "operation": "sum", "question": "Measure the total balance present in deliveries.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT SUM(balance) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(balance) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1e54bf2ef5c5da89e926_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "having", "question": "Display category groups in enrollments where row count is > 1500 and show the counts.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM enrollments GROUP BY category HAVING COUNT(*) > 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM enrollments GROUP BY category HAVING COUNT(*) > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "36117416028ec9713345_en", "language": "en", "backend": "supabase", "operation": "lte", "question": "Show the deliveries data where salary is at most 10000.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE salary <= 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE salary <= 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ab37d15a3c7b3a27c08f_en", "language": "en", "backend": "supabase", "operation": "or", "question": "Select rows from repairs where status is South or active.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE status IN ('South', 'active');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE status IN ('South', 'active');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ab37d15a3c7b3a27c08f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "or", "question": "repairs se status South ya active wali rows le lo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE status IN ('South', 'active');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE status IN ('South', 'active');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8d65ec475048a7f3c9da_en", "language": "en", "backend": "supabase", "operation": "sort_asc", "question": "Extract all rows from enrollments and list them by salary in ascending order.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments ORDER BY salary ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments ORDER BY salary ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8d65ec475048a7f3c9da_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sort_asc", "question": "enrollments ki saari rows salary ke kam se zyada order mein sort karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments ORDER BY salary ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments ORDER BY salary ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ad6bf3f875b067153767_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "count_eq", "question": "help me count rows in repairs where category is South", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM repairs WHERE category = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM repairs WHERE category = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ad6bf3f875b067153767_hinglish", "language": "hinglish", "backend": "supabase", "operation": "count_eq", "question": "repairs mein category South wale data ki sankhya batayo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM repairs WHERE category = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM repairs WHERE category = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fed9927b6460e7bd91b2_en", "language": "en", "backend": "supabase", "operation": "sql_error", "question": "Please review the structure of repairs before retrying, as the last query failed on an unrecognized column.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_repairs", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_repairs\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "1e54bf2ef5c5da89e926_en", "language": "en", "backend": "supabase", "operation": "having", "question": "Identify the category groups in enrollments with more than 1500 rows and report their respective counts.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM enrollments GROUP BY category HAVING COUNT(*) > 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM enrollments GROUP BY category HAVING COUNT(*) > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1e54bf2ef5c5da89e926_hinglish", "language": "hinglish", "backend": "supabase", "operation": "having", "question": "enrollments se 1500 rows se zyada wale category groups ko unki row count ke saath dikhaiye.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM enrollments GROUP BY category HAVING COUNT(*) > 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM enrollments GROUP BY category HAVING COUNT(*) > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8d65ec475048a7f3c9da_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sort_asc", "question": "retrieve all rows from enrollments and put them in order by salary lowest first", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments ORDER BY salary ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments ORDER BY salary ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "82d8a1bd3ade0d9c4ea9_en", "language": "en", "backend": "supabase", "operation": "group_count", "question": "Sum up the rows for each region in repairs.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT region, COUNT(*) FROM repairs GROUP BY region;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM repairs GROUP BY region;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "82d8a1bd3ade0d9c4ea9_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "group_count", "question": "i need the total rows for each region in repairs", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT region, COUNT(*) FROM repairs GROUP BY region;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM repairs GROUP BY region;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "82d8a1bd3ade0d9c4ea9_hinglish", "language": "hinglish", "backend": "supabase", "operation": "group_count", "question": "repairs me har region ke rows ka count do.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT region, COUNT(*) FROM repairs GROUP BY region;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM repairs GROUP BY region;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e3314881b15c1b561692_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "having", "question": "Display region groups in rentals where row count is > 10000 and show the counts.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT region, COUNT(*) FROM rentals GROUP BY region HAVING COUNT(*) > 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM rentals GROUP BY region HAVING COUNT(*) > 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e3314881b15c1b561692_hinglish", "language": "hinglish", "backend": "supabase", "operation": "having", "question": "rentals se 10000 rows se zyada wale region groups ko unki row count ke saath dikhaiye.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT region, COUNT(*) FROM rentals GROUP BY region HAVING COUNT(*) > 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM rentals GROUP BY region HAVING COUNT(*) > 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5c1ca9682dbf9c4df2a1_hinglish", "language": "hinglish", "backend": "supabase", "operation": "null", "question": "deliveries mein item_name null hone wali rows lao.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6016d266fd2535bc4479_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "year", "question": "Select deliveries rows where order_date is in 2021.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE EXTRACT(YEAR FROM order_date) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE EXTRACT(YEAR FROM order_date) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9925b01d3b059dd22a13_en", "language": "en", "backend": "supabase", "operation": "or", "question": "Select rows from repairs where kind is Delhi or South.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE kind IN ('Delhi', 'South');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE kind IN ('Delhi', 'South');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9925b01d3b059dd22a13_hinglish", "language": "hinglish", "backend": "supabase", "operation": "or", "question": "repairs se kind Delhi ya South wali rows le lo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE kind IN ('Delhi', 'South');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE kind IN ('Delhi', 'South');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5ec7a0dca8e2804dbca9_hinglish", "language": "hinglish", "backend": "supabase", "operation": "not_null", "question": "enrollments ko filter karke sirf wo rows dikhao jahan item_name null nahi hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE item_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE item_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2ee0d715b288b4660747_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "contains", "question": "Bring rows from rentals where display_name has North, no case distinction.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE LOWER(display_name) LIKE LOWER('%North%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE LOWER(display_name) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f2eb08449a82d31f500e_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "missing_value", "question": "filter repairs by category, i have not specified a value to use", "expected": {"action": "clarify", "question": "Which value should category equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should category equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "178d19ec70691d22e07e_en", "language": "en", "backend": "supabase", "operation": "eq", "question": "Query repairs to show all rows where department is South.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE department = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE department = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "178d19ec70691d22e07e_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "eq", "question": "Show all rows in repairs where department is South ok?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE department = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE department = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dfcb57284d8b78bb40f4_en", "language": "en", "backend": "supabase", "operation": "lt", "question": "Present the data from repairs where rating is below 1500.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE rating < 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE rating < 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a02ef9a534a2fa5110ea_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "and", "question": "give me rows from deliveries where status is Electronics and salary exceeds 10000", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE status = 'Electronics' AND salary > 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE status = 'Electronics' AND salary > 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5a3bf4f1cec9d88270ce_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "year", "question": "Select repairs rows where recorded_at is in 2024.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE EXTRACT(YEAR FROM recorded_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE EXTRACT(YEAR FROM recorded_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "36117416028ec9713345_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lte", "question": "deliveries ka wo data dikhao jaha salary 10000 se zyada nahi hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE salary <= 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE salary <= 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ab37d15a3c7b3a27c08f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "or", "question": "Please show rows from repairs where status is either South or active.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE status IN ('South', 'active');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE status IN ('South', 'active');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e3314881b15c1b561692_en", "language": "en", "backend": "supabase", "operation": "having", "question": "Identify the region groups in rentals with more than 10000 rows and report their respective counts.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT region, COUNT(*) FROM rentals GROUP BY region HAVING COUNT(*) > 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM rentals GROUP BY region HAVING COUNT(*) > 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5fc77c83422c0e74572f_en", "language": "en", "backend": "supabase", "operation": "gt", "question": "Select the rows from deliveries where price is greater than 5000.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE price > 5000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE price > 5000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5fc77c83422c0e74572f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gt", "question": "look up the rows in deliveries where price is greater than 5000", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE price > 5000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE price > 5000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6016d266fd2535bc4479_en", "language": "en", "backend": "supabase", "operation": "year", "question": "Display deliveries entries where order_date has the year 2021.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE EXTRACT(YEAR FROM order_date) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE EXTRACT(YEAR FROM order_date) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "068560b98b4de1eee24e_en", "language": "en", "backend": "supabase", "operation": "contains", "question": "Bring up the records in rentals where description features active, without regard to case.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE LOWER(description) LIKE LOWER('%active%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE LOWER(description) LIKE LOWER('%active%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c62049419f9ed25ca747_en", "language": "en", "backend": "supabase", "operation": "project_eq", "question": "Output the display_name from deliveries where kind is Books.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT display_name FROM deliveries WHERE kind = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM deliveries WHERE kind = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c62049419f9ed25ca747_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project_eq", "question": "show me the display_name in deliveries where kind is Books", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT display_name FROM deliveries WHERE kind = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM deliveries WHERE kind = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c62049419f9ed25ca747_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project_eq", "question": "deliveries se display_name output karo jahan kind Books hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT display_name FROM deliveries WHERE kind = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM deliveries WHERE kind = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fd8484763b52ad2d53c1_en", "language": "en", "backend": "supabase", "operation": "gt", "question": "Select the rows from repairs where rating is greater than 1000.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE rating > 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE rating > 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fd8484763b52ad2d53c1_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gt", "question": "look up the rows in repairs where rating is greater than 1000", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE rating > 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE rating > 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9912a7207dd6e403b2d5_en", "language": "en", "backend": "supabase", "operation": "top", "question": "Present the top 3 records from repairs by descending price value.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs ORDER BY price DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs ORDER BY price DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9912a7207dd6e403b2d5_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "top", "question": "Show the top 3 entries in repairs by price desc order.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs ORDER BY price DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs ORDER BY price DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9912a7207dd6e403b2d5_hinglish", "language": "hinglish", "backend": "supabase", "operation": "top", "question": "repairs mein price ke highest values wali 3 rows display karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs ORDER BY price DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs ORDER BY price DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2e35310942114f57ebb2_hinglish", "language": "hinglish", "backend": "supabase", "operation": "eq", "question": "repairs se query karo aur wo saari rows dikhao jahan category Books hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE category = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE category = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d1d96cfbe28ba036d11d_hi", "language": "hi", "backend": "supabase", "operation": "write", "question": "rentals से हर एक पंक्ति को निकालें।", "expected": {"action": "clarify", "question": "यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "dfcb57284d8b78bb40f4_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lt", "question": "output the records from repairs where rating is below 1500 please.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE rating < 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE rating < 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4658c817cf80ab71fd79_hi", "language": "hi", "backend": "supabase", "operation": "max", "question": "rentals में rating की अधिकतम संख्या क्या है?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT MAX(rating) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(rating) FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5a3bf4f1cec9d88270ce_en", "language": "en", "backend": "supabase", "operation": "year", "question": "Display repairs entries where recorded_at has the year 2024.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE EXTRACT(YEAR FROM recorded_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE EXTRACT(YEAR FROM recorded_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a8b04829d8a55fb6e012_en", "language": "en", "backend": "supabase", "operation": "date_after", "question": "Query rentals for rows where recorded_at is at or after 2024-03-15.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE recorded_at >= '2024-03-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE recorded_at >= '2024-03-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a8b04829d8a55fb6e012_hinglish", "language": "hinglish", "backend": "supabase", "operation": "date_after", "question": "rentals se wo records dikhao jahan recorded_at 2024-03-15 se pehle nahi hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE recorded_at >= '2024-03-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE recorded_at >= '2024-03-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5fc77c83422c0e74572f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gt", "question": "Mujhe deliveries se woh rows chahiye jahan price 5000 se zyada hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE price > 5000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE price > 5000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5c1ca9682dbf9c4df2a1_en", "language": "en", "backend": "supabase", "operation": "null", "question": "Show me the rows from deliveries where item_name is null.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f39ec09a8da50e72bdd5_hi", "language": "hi", "backend": "supabase", "operation": "min", "question": "enrollments में price का न्यूनतम स्तर क्या है?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT MIN(price) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(price) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6016d266fd2535bc4479_hinglish", "language": "hinglish", "backend": "supabase", "operation": "year", "question": "2021 ke order_date ke hisaab se deliveries se data nikalo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE EXTRACT(YEAR FROM order_date) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE EXTRACT(YEAR FROM order_date) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fd8484763b52ad2d53c1_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gt", "question": "Mujhe repairs se woh rows chahiye jahan rating 1000 se zyada hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE rating > 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE rating > 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9925b01d3b059dd22a13_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "or", "question": "Please show rows from repairs where kind is either Delhi or South.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE kind IN ('Delhi', 'South');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE kind IN ('Delhi', 'South');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2ee0d715b288b4660747_hinglish", "language": "hinglish", "backend": "supabase", "operation": "contains", "question": "Bring karo rentals se un rows ko jinka display_name North hai, case difference ke bina.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE LOWER(display_name) LIKE LOWER('%North%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE LOWER(display_name) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "99327f4048c296f053ea_en", "language": "en", "backend": "supabase", "operation": "date_after", "question": "Query enrollments for rows where created_at is at or after 2024-12-15.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE created_at >= '2024-12-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE created_at >= '2024-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0676be3ca0df76c86a56_hinglish", "language": "hinglish", "backend": "supabase", "operation": "month", "question": "repairs mein order_date ka month 12 hone wali rows dikhao.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE EXTRACT(MONTH FROM order_date) = 12;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE EXTRACT(MONTH FROM order_date) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a02ef9a534a2fa5110ea_en", "language": "en", "backend": "supabase", "operation": "and", "question": "Select rows from deliveries where status is Electronics and salary is greater than 10000.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE status = 'Electronics' AND salary > 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE status = 'Electronics' AND salary > 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f826d550b5dd9bf82144_hi", "language": "hi", "backend": "supabase", "operation": "avg", "question": "repairs में rating का औसत क्या है?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT AVG(rating) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(rating) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5a3bf4f1cec9d88270ce_hinglish", "language": "hinglish", "backend": "supabase", "operation": "year", "question": "2024 ke recorded_at ke hisaab se repairs se data nikalo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE EXTRACT(YEAR FROM recorded_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE EXTRACT(YEAR FROM recorded_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5c1ca9682dbf9c4df2a1_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "null", "question": "I need to check rows in deliveries where item_name is null.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f551f3aa4afc6f91a2d5_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "having", "question": "Display category groups in repairs where row count is > 100 and show the counts.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "de15f8143029f00970f6_en", "language": "en", "backend": "supabase", "operation": "bottom", "question": "Pull the 1 rows with the lowest price scores from deliveries, smallest to largest.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries ORDER BY price ASC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries ORDER BY price ASC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "99327f4048c296f053ea_hinglish", "language": "hinglish", "backend": "supabase", "operation": "date_after", "question": "enrollments se wo records dikhao jahan created_at 2024-12-15 se pehle nahi hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE created_at >= '2024-12-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE created_at >= '2024-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0676be3ca0df76c86a56_en", "language": "en", "backend": "supabase", "operation": "month", "question": "Provide rows from repairs where order_date's month value is 12.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE EXTRACT(MONTH FROM order_date) = 12;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE EXTRACT(MONTH FROM order_date) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0676be3ca0df76c86a56_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "month", "question": "give me the rows in repairs where month of order_date is 12", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE EXTRACT(MONTH FROM order_date) = 12;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE EXTRACT(MONTH FROM order_date) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "178d19ec70691d22e07e_hinglish", "language": "hinglish", "backend": "supabase", "operation": "eq", "question": "repairs se query karo aur wo saari rows dikhao jahan department South hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE department = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE department = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dfcb57284d8b78bb40f4_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lt", "question": "repairs se un rows ko show karo jahan rating 1500 se kam hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE rating < 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE rating < 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a8b04829d8a55fb6e012_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "date_after", "question": "Show me rentals data where recorded_at is not earlier than 2024-03-15.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE recorded_at >= '2024-03-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE recorded_at >= '2024-03-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "39205adac55e98cd6749_hi", "language": "hi", "backend": "supabase", "operation": "project_eq", "question": "rentals से display_name आउटपुट करें जहाँ kind O'Reilly है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT display_name FROM rentals WHERE kind = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM rentals WHERE kind = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f551f3aa4afc6f91a2d5_en", "language": "en", "backend": "supabase", "operation": "having", "question": "Identify the category groups in repairs with more than 100 rows and report their respective counts.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f551f3aa4afc6f91a2d5_hinglish", "language": "hinglish", "backend": "supabase", "operation": "having", "question": "repairs se 100 rows se zyada wale category groups ko unki row count ke saath dikhaiye.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1c48c9c4894db5982e9d_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sql_error", "question": "You had a unknown column error in the last query. Check rentals schema before trying again.", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "rentals"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"rentals\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "99327f4048c296f053ea_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "date_after", "question": "Show me enrollments data where created_at is not earlier than 2024-12-15.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE created_at >= '2024-12-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE created_at >= '2024-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f2eb08449a82d31f500e_en", "language": "en", "backend": "supabase", "operation": "missing_value", "question": "Please filter the records in repairs by category, but I have not specified a value.", "expected": {"action": "clarify", "question": "Which value should category equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should category equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "a02ef9a534a2fa5110ea_hinglish", "language": "hinglish", "backend": "supabase", "operation": "and", "question": "deliveries se wo rows display karo jahan status Electronics hai aur salary 10000 se zyada hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE status = 'Electronics' AND salary > 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE status = 'Electronics' AND salary > 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d2ac91503b4c4b8c4bbf_hi", "language": "hi", "backend": "supabase", "operation": "max", "question": "enrollments में quantity की अधिकतम संख्या क्या है?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT MAX(quantity) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(quantity) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2ee0d715b288b4660747_en", "language": "en", "backend": "supabase", "operation": "contains", "question": "Bring up the records in rentals where display_name features North, without regard to case.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE LOWER(display_name) LIKE LOWER('%North%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE LOWER(display_name) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c23d51b5c96bb4b300bf_hi", "language": "hi", "backend": "supabase", "operation": "group_count", "question": "rentals में प्रत्येक status के पंक्ति काउंट दें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT status, COUNT(*) FROM rentals GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT status, COUNT(*) FROM rentals GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8e0d860cde6e650084c2_hi", "language": "hi", "backend": "supabase", "operation": "or", "question": "enrollments से kind Electronics या Mumbai वाली पंक्तियाँ लें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE kind IN ('Electronics', 'Mumbai');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE kind IN ('Electronics', 'Mumbai');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a81aea4a2431376817c8_hi", "language": "hi", "backend": "supabase", "operation": "or", "question": "enrollments से kind Mumbai या Mumbai वाली पंक्तियाँ लें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE kind IN ('Mumbai', 'Mumbai');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE kind IN ('Mumbai', 'Mumbai');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "99c40dbbcc3778d58436_hi", "language": "hi", "backend": "supabase", "operation": "group_count", "question": "rentals में प्रत्येक status के पंक्ति काउंट दें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT status, COUNT(*) FROM rentals GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT status, COUNT(*) FROM rentals GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d00f22da639cc816ec58_en", "language": "en", "backend": "supabase", "operation": "final", "question": "The database tool gave 114 rows. What is the count?", "expected": {"action": "answer", "text": "The tool returned 114 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 114 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "d00f22da639cc816ec58_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "final", "question": "the database tool gave 114 rows. how many rows is that?", "expected": {"action": "answer", "text": "The tool returned 114 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 114 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "7a57ce9685f252c72f44_hi", "language": "hi", "backend": "supabase", "operation": "lt", "question": "rentals में वे पंक्तियाँ खोजें जहाँ quantity 3 से कम है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE quantity < 3;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE quantity < 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "de15f8143029f00970f6_hinglish", "language": "hinglish", "backend": "supabase", "operation": "bottom", "question": "deliveries se price ke hisaab se 1 minimum rows, lowest value se shuru karte hue, display karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries ORDER BY price ASC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries ORDER BY price ASC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bff6b3ccadd4c292165c_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sql_error", "question": "You had a unknown column error in the last query. Check deliveries schema before trying again.", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "deliveries"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"deliveries\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "f12c053fdb90a634ce20_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sql_error", "question": "You had a unknown column error in the last query. Check repairs schema before trying again.", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "repairs"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"repairs\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "64405609b3558aed49bd_en", "language": "en", "backend": "supabase", "operation": "date_after", "question": "Query deliveries for rows where order_date is at or after 2025-01-15.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE order_date >= '2025-01-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE order_date >= '2025-01-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d00f22da639cc816ec58_hinglish", "language": "hinglish", "backend": "supabase", "operation": "final", "question": "Database tool ne 114 rows laayi hain. Kripya sankhya batao.", "expected": {"action": "answer", "text": "Tool ne 114 rows return ki."}, "output": "{\"action\":\"answer\",\"text\":\"Tool ne 114 rows return ki.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "1c48c9c4894db5982e9d_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sql_error", "question": "Kyunki aakhri query unknown column ki wajah se fail hui, rentals schema inspect karne ke baad hi dobara try karo.", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "rentals"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"rentals\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "f2eb08449a82d31f500e_hinglish", "language": "hinglish", "backend": "supabase", "operation": "missing_value", "question": "Please repairs mein records ko category ke hisaab se filter karo, lekin maine koi value specify nahi ki hai.", "expected": {"action": "clarify", "question": "category ki kaunsi value chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"category ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "b6bd83761ee381e1df8c_hi", "language": "hi", "backend": "supabase", "operation": "max", "question": "repairs में balance की अधिकतम संख्या क्या है?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT MAX(balance) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(balance) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a2ad743e655e7c8923dc_hi", "language": "hi", "backend": "supabase", "operation": "project", "question": "मुझे rentals का customer_name चाहिए, बाकी कुछ नहीं।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT customer_name FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "da7c6370afcc35f790f8_hi", "language": "hi", "backend": "supabase", "operation": "sum", "question": "rentals एंटीटी के लिए quantity का कुल आउटपुट दें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT SUM(quantity) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(quantity) FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7782f04c53bb2061153a_hi", "language": "hi", "backend": "supabase", "operation": "lt", "question": "rentals में वे पंक्तियाँ खोजें जहाँ amount 1000 से कम है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE amount < 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE amount < 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "64405609b3558aed49bd_hinglish", "language": "hinglish", "backend": "supabase", "operation": "date_after", "question": "deliveries se wo records dikhao jahan order_date 2025-01-15 se pehle nahi hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE order_date >= '2025-01-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE order_date >= '2025-01-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c62049419f9ed25ca747_hi", "language": "hi", "backend": "supabase", "operation": "project_eq", "question": "deliveries से display_name आउटपुट करें जहाँ kind Books है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT display_name FROM deliveries WHERE kind = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM deliveries WHERE kind = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "efee860c507a44ecafbb_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sql_error", "question": "You had a unknown column error in the last query. Check deliveries schema before trying again.", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "deliveries"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"deliveries\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "36d48e67b84ab1dd10fa_hi", "language": "hi", "backend": "supabase", "operation": "project", "question": "मुझे enrollments का display_name चाहिए, बाकी कुछ नहीं।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT display_name FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "64405609b3558aed49bd_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "date_after", "question": "Show me deliveries data where order_date is not earlier than 2025-01-15.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE order_date >= '2025-01-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE order_date >= '2025-01-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "928e148fdcacd00b97f8_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sql_error", "question": "You had a unknown column error in the last query. Check deliveries schema before trying again.", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "deliveries"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"deliveries\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "ab37d15a3c7b3a27c08f_hi", "language": "hi", "backend": "supabase", "operation": "or", "question": "repairs से status South या active वाली पंक्तियाँ लें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE status IN ('South', 'active');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE status IN ('South', 'active');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "327bced9ccce8adafb67_hi", "language": "hi", "backend": "supabase", "operation": "lt", "question": "enrollments में वे पंक्तियाँ खोजें जहाँ price 20 से कम है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE price < 20;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE price < 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "de6c0f1e7617ba0be5cc_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "write", "question": "Clear enrollments completely.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "d169f3070176e20c972f_hi", "language": "hi", "backend": "supabase", "operation": "ambiguous", "question": "deliveries के सर्वोत्तम रिकॉर्ड उजागर करें।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "1c48c9c4894db5982e9d_en", "language": "en", "backend": "mysql", "operation": "sql_error", "question": "Please review the structure of rentals before retrying, as the last query failed on an unrecognized column.", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "rentals"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"rentalsschemazed\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "9925b01d3b059dd22a13_hi", "language": "hi", "backend": "supabase", "operation": "or", "question": "repairs से kind Delhi या South वाली पंक्तियाँ लें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE kind IN ('Delhi', 'South');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE kind IN ('Delhi', 'South');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4cb74692efd909e4f3fd_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "count", "question": "count the rows in enrollments", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT COUNT(*) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT COUNT(*) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bff6b3ccadd4c292165c_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sql_error", "question": "Kyunki aakhri query unknown column ki wajah se fail hui, deliveries schema inspect karne ke baad hi dobara try karo.", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "deliveries"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"deliveries\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "40a4c97e6c006d30b593_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "count", "question": "count the rows in enrollments", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT COUNT(*) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT COUNT(*) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "780db517b686cfd2fce3_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "list_tables", "question": "list out the tables in this database", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_rentals", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_rentals\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "f12c053fdb90a634ce20_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sql_error", "question": "Kyunki aakhri query unknown column ki wajah se fail hui, repairs schema inspect karne ke baad hi dobara try karo.", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "repairs"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"repairs\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "4de9a85576f4fbfac7d2_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "all", "question": "pull all rows from rentals", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_rentals", "query": "SELECT * FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_rentals\",\"query\":\"SELECT * FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "50120f3b3fbdc0e48064_en", "language": "en", "backend": "supabase", "operation": "final", "question": "The database tool gave 143 rows. What is the count?", "expected": {"action": "answer", "text": "The tool returned 143 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 143 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "50120f3b3fbdc0e48064_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "final", "question": "the database tool gave 143 rows. how many rows is that?", "expected": {"action": "answer", "text": "The tool returned 143 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 143 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "780db517b686cfd2fce3_en", "language": "en", "backend": "supabase", "operation": "list_tables", "question": "Fetch the table list for this database.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_rentals", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_rentals\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "780db517b686cfd2fce3_hinglish", "language": "hinglish", "backend": "supabase", "operation": "list_tables", "question": "Kripya is database ki tables fetch karo.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_rentals", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_rentals\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "236309e4fd1627100f3a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sum", "question": "please find the total score in enrollments", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT SUM(score) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT SUM(score) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "efee860c507a44ecafbb_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sql_error", "question": "Kyunki aakhri query unknown column ki wajah se fail hui, deliveries schema inspect karne ke baad hi dobara try karo.", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "deliveries"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"dekhriveries\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "e971c6c1ded7274f9394_hi", "language": "hi", "backend": "supabase", "operation": "sum", "question": "deliveries एंटीटी के लिए balance का कुल आउटपुट दें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT SUM(balance) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(balance) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f638660fca2c9a9c9386_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "max", "question": "get the maximum salary from enrollments please", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT MAX(salary) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT MAX(salary) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4de9a85576f4fbfac7d2_hinglish", "language": "hinglish", "backend": "supabase", "operation": "all", "question": "rentals se saare rows le aao.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_rentals", "query": "SELECT * FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_rentals\",\"query\":\"SELECT * FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "50120f3b3fbdc0e48064_hinglish", "language": "hinglish", "backend": "supabase", "operation": "final", "question": "Database tool ne 143 rows laayi hain. Kripya sankhya batao.", "expected": {"action": "answer", "text": "Tool ne 143 rows return ki."}, "output": "{\"action\":\"answer\",\"text\":\"Tool ne 143 rows return ki.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "928e148fdcacd00b97f8_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sql_error", "question": "Kyunki aakhri query unknown column ki wajah se fail hui, deliveries schema inspect karne ke baad hi dobara try karo.", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "deliveries"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"dekhriveries\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "6016d266fd2535bc4479_hi", "language": "hi", "backend": "supabase", "operation": "year", "question": "2021 के order_date के आधार पर deliveries से डेटा निकालें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE EXTRACT(YEAR FROM order_date) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE EXTRACT(YEAR FROM order_date) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fe9e519448ad73aea083_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "list_tables", "question": "list out the tables in this database", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_deliveries", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "494e7ce9785848f5299f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "describe", "question": "Display the field types and names for rentals.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_rentals", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_rentals\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "494e7ce9785848f5299f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "describe", "question": "Display the columns and data types for rentals.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_rentals", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_rentals\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "e27bab397ec7daf2a101_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "distinct", "question": "find the unique kind in enrollments", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT DISTINCT kind FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT DISTINCT kind FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "de6c0f1e7617ba0be5cc_hinglish", "language": "hinglish", "backend": "supabase", "operation": "write", "question": "enrollments se har ek row remove karo.", "expected": {"action": "clarify", "question": "Yeh connection read-only hai. Kya SELECT query chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"Yeh connection read-only hai. Kya SELECT query chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "f638660fca2c9a9c9386_en", "language": "en", "backend": "supabase", "operation": "max", "question": "Give me the highest salary for enrollments.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT MAX(salary) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT MAX(salary) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "46c2ec98a037d78a9576_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "list_tables", "question": "list out the tables in this database", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_deliveries", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "bff6b3ccadd4c292165c_en", "language": "en", "backend": "mysql", "operation": "sql_error", "question": "Please review the structure of deliveries before retrying, as the last query failed on an unrecognized column.", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "deliveries"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"deliveries\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "4de9a85576f4fbfac7d2_en", "language": "en", "backend": "supabase", "operation": "all", "question": "Bring up all rows from rentals.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_rentals", "query": "SELECT * FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_rentals\",\"query\":\"SELECT * FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a509f9b46620c564c0cf_hi", "language": "hi", "backend": "supabase", "operation": "missing_schema", "question": "मेने डेटाबेस स्कीमा प्रदान नहीं की है, इसलिए repairs के लिए SQL लिखने से पहले उपलब्ध तालिकाओं को खोजें।", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_repairs", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_repairs\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "f12c053fdb90a634ce20_en", "language": "en", "backend": "mysql", "operation": "sql_error", "question": "Please review the structure of repairs before retrying, as the last query failed on an unrecognized column.", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "repairs"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"repairs\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "5a3bf4f1cec9d88270ce_hi", "language": "hi", "backend": "supabase", "operation": "year", "question": "2024 के recorded_at के आधार पर repairs से डेटा निकालें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE EXTRACT(YEAR FROM recorded_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE EXTRACT(YEAR FROM recorded_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fe9e519448ad73aea083_en", "language": "en", "backend": "supabase", "operation": "list_tables", "question": "Fetch the table list for this database.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_deliveries", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "fe9e519448ad73aea083_hinglish", "language": "hinglish", "backend": "supabase", "operation": "list_tables", "question": "Kripya is database ki tables fetch karo.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_deliveries", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "9b02b7583735a1bdd0f2_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ambiguous", "question": "Show the leading records in enrollments.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "8ecb63601a1ad54fe8ff_hinglish", "language": "hinglish", "backend": "supabase", "operation": "null", "question": "enrollments mein name null hone wali rows lao.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4cb74692efd909e4f3fd_en", "language": "en", "backend": "supabase", "operation": "count", "question": "List the number of rows found in enrollments.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT COUNT(*) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT COUNT(*) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f638660fca2c9a9c9386_hinglish", "language": "hinglish", "backend": "supabase", "operation": "max", "question": "enrollments mein salary ki maximum number kya hai?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT MAX(salary) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT MAX(salary) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "04cdd3f064826214f853_hi", "language": "hi", "backend": "supabase", "operation": "distinct", "question": "enrollments में मौजूद डेटा से status के विशिष्ट मान निकालें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT DISTINCT status FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT status FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "46c2ec98a037d78a9576_en", "language": "en", "backend": "supabase", "operation": "list_tables", "question": "Fetch the table list for this database.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_deliveries", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "46c2ec98a037d78a9576_hinglish", "language": "hinglish", "backend": "supabase", "operation": "list_tables", "question": "Kripya is database ki tables fetch karo.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_deliveries", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "7513979dd6549473de83_hi", "language": "hi", "backend": "supabase", "operation": "count_eq", "question": "rentals में city active होने की स्थिति में पंक्तियों की गिनती करें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM rentals WHERE city = 'active';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM rentals WHERE city = 'active';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dfcb57284d8b78bb40f4_hi", "language": "hi", "backend": "supabase", "operation": "lt", "question": "repairs में वे पंक्तियाँ खोजें जहाँ rating 1500 से कम है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE rating < 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE rating < 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "40a4c97e6c006d30b593_en", "language": "en", "backend": "supabase", "operation": "count", "question": "List the number of rows found in enrollments.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT COUNT(*) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT COUNT(*) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a7d907a9a18c081961a8_hi", "language": "hi", "backend": "supabase", "operation": "distinct", "question": "enrollments में मौजूद डेटा से status के विशिष्ट मान निकालें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT DISTINCT status FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT status FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c3c868696c82e387acd8_hi", "language": "hi", "backend": "supabase", "operation": "gte", "question": "enrollments की वे एंट्री प्राप्त करें जहाँ balance 0 या उससे ऊपर है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE balance >= 0;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE balance >= 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "36117416028ec9713345_hi", "language": "hi", "backend": "supabase", "operation": "lte", "question": "deliveries का वो डेटा दिखाएं जहां salary 10000 से अधिक नहीं है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE salary <= 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE salary <= 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "cdc7d96784e2e9f3c94b_hi", "language": "hi", "backend": "supabase", "operation": "distinct", "question": "rentals में मौजूद डेटा से department के विशिष्ट मान निकालें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT DISTINCT department FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c4e9855d27b398c547a8_hi", "language": "hi", "backend": "supabase", "operation": "top", "question": "rentals में balance के उच्चतम मानों वाली 1 पंक्तियाँ प्रदर्शित करें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals ORDER BY balance DESC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals ORDER BY balance DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "82d8a1bd3ade0d9c4ea9_hi", "language": "hi", "backend": "supabase", "operation": "group_count", "question": "repairs में प्रत्येक region के पंक्ति काउंट दें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT region, COUNT(*) FROM repairs GROUP BY region;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM repairs GROUP BY region;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dbb8cd8445dd19a458e2_hi", "language": "hi", "backend": "supabase", "operation": "eq", "question": "enrollments से पूछें और वे सभी पंक्तियाँ दिखाएँ जहाँ department Pune है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE department = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE department = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5c1ca9682dbf9c4df2a1_hi", "language": "hi", "backend": "supabase", "operation": "null", "question": "deliveries में item_name null होने वाली पंक्तियाँ लाएँ।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "236309e4fd1627100f3a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sum", "question": "enrollments mein score ka total measure karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT SUM(score) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT SUM(score) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b9e6b86227c7663bd9bb_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "describe", "question": "Display the field types and names for rentals.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_rentals", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_rentals\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "b9e6b86227c7663bd9bb_hinglish", "language": "hinglish", "backend": "supabase", "operation": "describe", "question": "Display the columns and data types for rentals.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_rentals", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_rentals\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "efee860c507a44ecafbb_en", "language": "en", "backend": "mysql", "operation": "sql_error", "question": "Please review the structure of deliveries before retrying, as the last query failed on an unrecognized column.", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "deliveries"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"deliveries\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "de6c0f1e7617ba0be5cc_en", "language": "en", "backend": "supabase", "operation": "write", "question": "Blow away all rows from enrollments.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}}
{"id": "2a70571066edacbb19ac_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "bottom", "question": "give me the 5 lowest score data from rentals ordered ascending", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_rentals", "query": "SELECT * FROM rentals ORDER BY score ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_rentals\",\"query\":\"SELECT * FROM rentals ORDER BY score ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f805b47d834fac71679e_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "join", "question": "retrieve rentals.customer_name and rentals_groups.label join rentals.group_id to rentals_groups.id", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT rentals.customer_name, rentals_groups.label FROM rentals JOIN rentals_groups ON rentals.group_id = rentals_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT rentals.customer_name, rentals_groups.label FROM rentals JOIN rentals_groups ON rentals.group_id = rentals_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "928e148fdcacd00b97f8_en", "language": "en", "backend": "mysql", "operation": "sql_error", "question": "Please review the structure of deliveries before retrying, as the last query failed on an unrecognized column.", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "deliveries"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"deliveries\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "9bb785183a3dc77c2cfa_hi", "language": "hi", "backend": "supabase", "operation": "group_sum", "question": "मुझे enrollments में प्रत्येक status के लिए balance का कुल दिखाना है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT status, SUM(balance) FROM enrollments GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT status, SUM(balance) FROM enrollments GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9b0921e93df3d40ae856_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "bottom", "question": "give me the 1 lowest balance data from enrollments ordered ascending", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments ORDER BY balance ASC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments ORDER BY balance ASC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ccbdc5b0dcc9fcc00608_hinglish", "language": "hinglish", "backend": "supabase", "operation": "month", "question": "rentals mein recorded_at ka month 10 hone wali rows dikhao.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_rentals", "query": "SELECT * FROM rentals WHERE EXTRACT(MONTH FROM recorded_at) = 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_rentals\",\"query\":\"SELECT * FROM rentals WHERE EXTRACT(MONTH FROM recorded_at) = 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "236309e4fd1627100f3a_en", "language": "en", "backend": "supabase", "operation": "sum", "question": "Measure the total score present in enrollments.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT SUM(score) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT SUM(score) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9b02b7583735a1bdd0f2_en", "language": "en", "backend": "supabase", "operation": "ambiguous", "question": "Reveal the leading records in enrollments.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "9b02b7583735a1bdd0f2_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ambiguous", "question": "enrollments ke best records reveal karo.", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "8ecb63601a1ad54fe8ff_en", "language": "en", "backend": "supabase", "operation": "null", "question": "Show me the rows from enrollments where name is null.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4cb74692efd909e4f3fd_hinglish", "language": "hinglish", "backend": "supabase", "operation": "count", "question": "enrollments ki rows ka volume kya hai?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT COUNT(*) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "4c61d39f7f226ce5244f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "year", "question": "Select enrollments rows where start_date is in 2025.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE EXTRACT(YEAR FROM start_date) = 2025;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE EXTRACT(YEAR FROM start_date) = 2025;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "eac1735177916d6d370c_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "join", "question": "retrieve enrollments.display_name and enrollments_groups.label join enrollments.group_id to enrollments_groups.id", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT enrollments.display_name, enrollments_groups.label FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT enrollments.display_name, enrollments_groups.label FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ee24cd456d1464d72bf4_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "write", "question": "Clear repairs completely.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "15e384ce46e890333204_hi", "language": "hi", "backend": "supabase", "operation": "gt", "question": "मुझे enrollments से वे पंक्तियाँ चाहिए जहाँ rating 500 से अधिक है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE rating > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE rating > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9adcc0998325a7b1eba4_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lte", "question": "select rentals rows where quantity is not more than 100", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_rentals", "query": "SELECT * FROM rentals WHERE quantity <= 100;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_rentals\",\"query\":\"SELECT * FROM rentals WHERE quantity <= 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "40a4c97e6c006d30b593_hinglish", "language": "hinglish", "backend": "supabase", "operation": "count", "question": "enrollments ki rows ka volume kya hai?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT COUNT(*) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "17da76d72f74206f6d3b_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "bottom", "question": "give me the 20 lowest salary data from enrollments ordered ascending", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments ORDER BY salary ASC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments ORDER BY salary ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b8697f5a0689a023e448_hi", "language": "hi", "backend": "supabase", "operation": "month", "question": "enrollments में created_at 10 महीने में होने वाली पंक्तियां दिखाएं।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE EXTRACT(MONTH FROM created_at) = 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE EXTRACT(MONTH FROM created_at) = 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e6b1d053d0a307dba955_hinglish", "language": "hinglish", "backend": "supabase", "operation": "or", "question": "rentals se department Books ya North wali rows le lo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_rentals", "query": "SELECT * FROM rentals WHERE department IN ('Books', 'North');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_rentals\",\"query\":\"SELECT * FROM rentals WHERE department IN ('Books', 'North');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bc42ac714674eb7e9508_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sum", "question": "please find the total quantity in repairs", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT SUM(quantity) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT SUM(quantity) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a7342b948d7bd031d82c_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "min", "question": "Show me minimum cost in deliveries.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT MIN(cost) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT MIN(cost) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3cfa2682031af21eedd8_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project", "question": "bring up name from deliveries only", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT name FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT name FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3cfa2682031af21eedd8_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project", "question": "i want just name from deliveries ok", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT name FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT name FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8ecb63601a1ad54fe8ff_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "null", "question": "I need to check rows in enrollments where name is null.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f0c710492ac60feed37b_en", "language": "en", "backend": "supabase", "operation": "group_count", "question": "Sum up the rows for each region in enrollments.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT region, COUNT(*) FROM enrollments GROUP BY region;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT region, COUNT(*) FROM enrollments GROUP BY region;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f0c710492ac60feed37b_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "group_count", "question": "i need the total rows for each region in enrollments", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT region, COUNT(*) FROM enrollments GROUP BY region;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT region, COUNT(*) FROM enrollments GROUP BY region;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f0c710492ac60feed37b_hinglish", "language": "hinglish", "backend": "supabase", "operation": "group_count", "question": "enrollments me har region ke rows ka count do.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT region, COUNT(*) FROM enrollments GROUP BY region;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT region, COUNT(*) FROM enrollments GROUP BY region;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4c61d39f7f226ce5244f_en", "language": "en", "backend": "supabase", "operation": "year", "question": "Display enrollments entries where start_date has the year 2025.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE EXTRACT(YEAR FROM start_date) = 2025;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE EXTRACT(YEAR FROM start_date) = 2025;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9adcc0998325a7b1eba4_en", "language": "en", "backend": "supabase", "operation": "lte", "question": "Show the rentals data where quantity is at most 100.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_rentals", "query": "SELECT * FROM rentals WHERE quantity <= 100;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_rentals\",\"query\":\"SELECT * FROM rentals WHERE quantity <= 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e6b1d053d0a307dba955_en", "language": "en", "backend": "supabase", "operation": "or", "question": "Select rows from rentals where department is Books or North.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_rentals", "query": "SELECT * FROM rentals WHERE department IN ('Books', 'North');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_rentals\",\"query\":\"SELECT * FROM rentals WHERE department IN ('Books', 'North');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f216d01e1724e4ca0e48_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project", "question": "bring up name from deliveries only", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT name FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT name FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f216d01e1724e4ca0e48_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project", "question": "i want just name from deliveries ok", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT name FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT name FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a8b04829d8a55fb6e012_hi", "language": "hi", "backend": "supabase", "operation": "date_after", "question": "rentals से वे रिकॉर्ड दिखाएं जहाँ recorded_at 2024-03-15 से पहले नहीं है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE recorded_at >= '2024-03-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE recorded_at >= '2024-03-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ccbdc5b0dcc9fcc00608_en", "language": "en", "backend": "supabase", "operation": "month", "question": "Provide rows from rentals where recorded_at's month value is 10.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_rentals", "query": "SELECT * FROM rentals WHERE EXTRACT(MONTH FROM recorded_at) = 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_rentals\",\"query\":\"SELECT * FROM rentals WHERE EXTRACT(MONTH FROM recorded_at) = 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ccbdc5b0dcc9fcc00608_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "month", "question": "give me the rows in rentals where month of recorded_at is 10", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_rentals", "query": "SELECT * FROM rentals WHERE EXTRACT(MONTH FROM recorded_at) = 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_rentals\",\"query\":\"SELECT * FROM rentals WHERE EXTRACT(MONTH FROM recorded_at) = 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "494e7ce9785848f5299f_en", "language": "en", "backend": "supabase", "operation": "describe", "question": "Fetch the metadata regarding columns and types for rentals.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_rentals", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_rentals\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a7342b948d7bd031d82c_en", "language": "en", "backend": "supabase", "operation": "min", "question": "Give me the minimum cost from deliveries.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT MIN(cost) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT MIN(cost) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2e35310942114f57ebb2_hi", "language": "hi", "backend": "supabase", "operation": "eq", "question": "repairs से पूछें और वे सभी पंक्तियाँ दिखाएँ जहाँ category Books है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE category = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE category = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "75b89b71dde15656327c_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "distinct", "question": "find the unique department in deliveries", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT DISTINCT department FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT DISTINCT department FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4c61d39f7f226ce5244f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "year", "question": "2025 ke start_date ke hisaab se enrollments se data nikalo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE EXTRACT(YEAR FROM start_date) = 2025;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE EXTRACT(YEAR FROM start_date) = 2025;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9adcc0998325a7b1eba4_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lte", "question": "rentals ka wo data dikhao jaha quantity 100 se zyada nahi hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_rentals", "query": "SELECT * FROM rentals WHERE quantity <= 100;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_rentals\",\"query\":\"SELECT * FROM rentals WHERE quantity <= 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1671c0540cd054060350_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "year", "question": "Select enrollments rows where recorded_at is in 2022.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE EXTRACT(YEAR FROM recorded_at) = 2022;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE EXTRACT(YEAR FROM recorded_at) = 2022;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "66a886154d6a2a05a7f7_hinglish", "language": "hinglish", "backend": "supabase", "operation": "null", "question": "repairs mein name null hone wali rows lao.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8488c3423e53e10c7853_hi", "language": "hi", "backend": "supabase", "operation": "count", "question": "repairs में स्टोर की गई पंक्तियों की संख्या क्या है?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a7342b948d7bd031d82c_hinglish", "language": "hinglish", "backend": "supabase", "operation": "min", "question": "deliveries mein cost ka minimum limit kya hai?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT MIN(cost) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT MIN(cost) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e27bab397ec7daf2a101_en", "language": "en", "backend": "supabase", "operation": "distinct", "question": "Pull out the distinct kind values from the data in enrollments.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT DISTINCT kind FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT DISTINCT kind FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e27bab397ec7daf2a101_hinglish", "language": "hinglish", "backend": "supabase", "operation": "distinct", "question": "enrollments mein maujood data se kind ke distinct values pull karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT DISTINCT kind FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT DISTINCT kind FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "95a0b0a0ed8d648db327_en", "language": "en", "backend": "supabase", "operation": "join_filter", "question": "Select enrollments.name from enrollments joined with enrollments_groups on enrollments.group_id where enrollments_groups.label is South.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT enrollments.name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT enrollments.name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "95a0b0a0ed8d648db327_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "join_filter", "question": "select enrollments.name from enrollments joined to enrollments_groups on enrollments.group_id where enrollments_groups.label is South please", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT enrollments.name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT enrollments.name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "99327f4048c296f053ea_hi", "language": "hi", "backend": "supabase", "operation": "date_after", "question": "enrollments से वे रिकॉर्ड दिखाएं जहाँ created_at 2024-12-15 से पहले नहीं है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE created_at >= '2024-12-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE created_at >= '2024-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f805b47d834fac71679e_en", "language": "en", "backend": "supabase", "operation": "join", "question": "Present rentals.customer_name and rentals_groups.label by associating rentals.group_id with rentals_groups.id.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT rentals.customer_name, rentals_groups.label FROM rentals JOIN rentals_groups ON rentals.group_id = rentals_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT rentals.customer_name, rentals_groups.label FROM rentals JOIN rentals_groups ON rentals.group_id = rentals_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f805b47d834fac71679e_hinglish", "language": "hinglish", "backend": "supabase", "operation": "join", "question": "rentals.group_id ko rentals_groups.id se join karke rentals.customer_name aur rentals_groups.label ka data dikhao.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT rentals.customer_name, rentals_groups.label FROM rentals JOIN rentals_groups ON rentals.group_id = rentals_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT rentals.customer_name, rentals_groups.label FROM rentals JOIN rentals_groups ON rentals.group_id = rentals_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ebf0786e1ce12c15da6f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "null", "question": "deliveries mein title null hone wali rows lao.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1671c0540cd054060350_en", "language": "en", "backend": "supabase", "operation": "year", "question": "Display enrollments entries where recorded_at has the year 2022.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE EXTRACT(YEAR FROM recorded_at) = 2022;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE EXTRACT(YEAR FROM recorded_at) = 2022;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e6b1d053d0a307dba955_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "or", "question": "Please show rows from rentals where department is either Books or North.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_rentals", "query": "SELECT * FROM rentals WHERE department IN ('Books', 'North');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_rentals\",\"query\":\"SELECT * FROM rentals WHERE department IN ('Books', 'North');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "70b8594259c7b8e10305_hinglish", "language": "hinglish", "backend": "supabase", "operation": "null", "question": "deliveries mein title null hone wali rows lao.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c49bd7e05e6964389dd5_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ambiguous", "question": "Show the leading records in deliveries.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "bc42ac714674eb7e9508_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sum", "question": "repairs mein quantity ka total measure karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT SUM(quantity) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT SUM(quantity) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c5a56e24a5878c81791a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gte", "question": "get rows from repairs where price is 100 or above", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE price >= 100;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE price >= 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fe5044da918260a617a9_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "group_sum", "question": "show the total cost for each status in repairs now", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT status, SUM(cost) FROM repairs GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT status, SUM(cost) FROM repairs GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b9e6b86227c7663bd9bb_en", "language": "en", "backend": "supabase", "operation": "describe", "question": "Fetch the metadata regarding columns and types for rentals.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_rentals", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_rentals\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "3cfa2682031af21eedd8_en", "language": "en", "backend": "supabase", "operation": "project", "question": "Let's look at name in deliveries specifically.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT name FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT name FROM deliveries WHERE city = 'Cetnaies';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "no such column: city"}}
{"id": "4c485cb9781970855140_hi", "language": "hi", "backend": "supabase", "operation": "sort_desc", "question": "enrollments से सभी पंक्तियाँ quantity के उच्चतम से निम्नतम क्रम में लौटाएँ।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments ORDER BY quantity DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments ORDER BY quantity DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2a70571066edacbb19ac_en", "language": "en", "backend": "supabase", "operation": "bottom", "question": "Pull the 5 rows with the lowest score scores from rentals, smallest to largest.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_rentals", "query": "SELECT * FROM rentals ORDER BY score ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_rentals\",\"query\":\"SELECT * FROM rentals ORDER BY score ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "178d19ec70691d22e07e_hi", "language": "hi", "backend": "supabase", "operation": "eq", "question": "repairs से पूछें और वे सभी पंक्तियाँ दिखाएँ जहाँ department South है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE department = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE department = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "69aa7071e087609fac06_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lte", "question": "select repairs rows where cost is not more than 10", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE cost <= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE cost <= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3d12ba589211e391881d_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "not_null", "question": "Show the repairs rows where title is not null.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE title IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE title IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "eac1735177916d6d370c_en", "language": "en", "backend": "supabase", "operation": "join", "question": "Present enrollments.display_name and enrollments_groups.label by associating enrollments.group_id with enrollments_groups.id.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT enrollments.display_name, enrollments_groups.label FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT enrollments.display_name, enrollments_groups.label FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ee24cd456d1464d72bf4_hinglish", "language": "hinglish", "backend": "supabase", "operation": "write", "question": "repairs se har ek row remove karo.", "expected": {"action": "clarify", "question": "Yeh connection read-only hai. Kya SELECT query chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"Yeh connection read-only hai. Kya SELECT query chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "1140aefb82deeef59b85_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sort_desc", "question": "show all repairs rows ordered by balance highest to lowest", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs ORDER BY balance DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs ORDER BY balance DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1671c0540cd054060350_hinglish", "language": "hinglish", "backend": "supabase", "operation": "year", "question": "2022 ke recorded_at ke hisaab se enrollments se data nikalo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE EXTRACT(YEAR FROM recorded_at) = 2022;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE EXTRACT(YEAR FROM recorded_at) = 2022;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5b11e37f42eb06caba31_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "not_null", "question": "Show the deliveries rows where description is not null.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f216d01e1724e4ca0e48_en", "language": "en", "backend": "supabase", "operation": "project", "question": "Let's look at name in deliveries specifically.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT name FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT name FROM deliveries WHERE city = 'Aetotal';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "no such column: city"}}
{"id": "bb6e110cb30376d23598_hi", "language": "hi", "backend": "supabase", "operation": "sort_desc", "question": "enrollments से सभी पंक्तियाँ price के उच्चतम से निम्नतम क्रम में लौटाएँ।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments ORDER BY price DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments ORDER BY price DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8d65ec475048a7f3c9da_hi", "language": "hi", "backend": "supabase", "operation": "sort_asc", "question": "enrollments की सभी पंक्तियाँ salary के कम से ज्यादा क्रम में क्रमबद्ध करें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments ORDER BY salary ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments ORDER BY salary ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5fc77c83422c0e74572f_hi", "language": "hi", "backend": "supabase", "operation": "gt", "question": "मुझे deliveries से वे पंक्तियाँ चाहिए जहाँ price 5000 से अधिक है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE price > 5000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE price > 5000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ad6bf3f875b067153767_hi", "language": "hi", "backend": "supabase", "operation": "count_eq", "question": "repairs में category South होने की स्थिति में पंक्तियों की गिनती करें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM repairs WHERE category = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM repairs WHERE category = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bc42ac714674eb7e9508_en", "language": "en", "backend": "supabase", "operation": "sum", "question": "Measure the total quantity present in repairs.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT SUM(quantity) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_reeass\",\"query\":\"SELECT SUM(quantity) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
{"id": "9b0921e93df3d40ae856_en", "language": "en", "backend": "supabase", "operation": "bottom", "question": "Pull the 1 rows with the lowest balance scores from enrollments, smallest to largest.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments ORDER BY balance ASC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments ORDER BY balance ASC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fd8484763b52ad2d53c1_hi", "language": "hi", "backend": "supabase", "operation": "gt", "question": "मुझे repairs से वे पंक्तियाँ चाहिए जहाँ rating 1000 से अधिक है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE rating > 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE rating > 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c5a56e24a5878c81791a_en", "language": "en", "backend": "supabase", "operation": "gte", "question": "Find records in repairs where price is not below 100.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE price >= 100;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE price >= 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "66a886154d6a2a05a7f7_en", "language": "en", "backend": "supabase", "operation": "null", "question": "Show me the rows from repairs where name is null.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0bdc9f428b0896d7e38a_en", "language": "en", "backend": "supabase", "operation": "sort_asc", "question": "Extract all rows from enrollments and list them by balance in ascending order.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments ORDER BY balance ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments ORDER BY balance ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0bdc9f428b0896d7e38a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sort_asc", "question": "enrollments ki saari rows balance ke kam se zyada order mein sort karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments ORDER BY balance ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments ORDER BY balance ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a1b7d1103527d4724102_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "describe", "question": "Display the field types and names for deliveries.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_deliveries", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a1b7d1103527d4724102_hinglish", "language": "hinglish", "backend": "supabase", "operation": "describe", "question": "Display the columns and data types for deliveries.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_deliveries", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "69aa7071e087609fac06_en", "language": "en", "backend": "supabase", "operation": "lte", "question": "Show the repairs data where cost is at most 10.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE cost <= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE cost <= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ebf0786e1ce12c15da6f_en", "language": "en", "backend": "supabase", "operation": "null", "question": "Show me the rows from deliveries where title is null.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c752dd15f0913fe63436_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "bottom", "question": "give me the 5 lowest cost data from repairs ordered ascending", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs ORDER BY cost ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs ORDER BY cost ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "eac1735177916d6d370c_hinglish", "language": "hinglish", "backend": "supabase", "operation": "join", "question": "enrollments.group_id ko enrollments_groups.id se join karke enrollments.display_name aur enrollments_groups.label ka data dikhao.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT enrollments.display_name, enrollments_groups.label FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT enrollments.display_name, enrollments_groups.label FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "17da76d72f74206f6d3b_en", "language": "en", "backend": "supabase", "operation": "bottom", "question": "Pull the 20 rows with the lowest salary scores from enrollments, smallest to largest.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments ORDER BY salary ASC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments ORDER BY salary ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "70b8594259c7b8e10305_en", "language": "en", "backend": "supabase", "operation": "null", "question": "Show me the rows from deliveries where title is null.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "619f53e951d6c4a5acdb_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "bottom", "question": "give me the 1 lowest quantity data from deliveries ordered ascending", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries ORDER BY quantity ASC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries ORDER BY quantity ASC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9d71fdd5c658b26d6b5f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "describe", "question": "Display the field types and names for repairs.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_repairs", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_repairs\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "9d71fdd5c658b26d6b5f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "describe", "question": "Display the columns and data types for repairs.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_repairs", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_repairs\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "c49bd7e05e6964389dd5_en", "language": "en", "backend": "supabase", "operation": "ambiguous", "question": "Reveal the leading records in deliveries.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "c49bd7e05e6964389dd5_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ambiguous", "question": "deliveries ke best records reveal karo.", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "c077714e2e488a364925_en", "language": "en", "backend": "supabase", "operation": "between", "question": "Output the rows from enrollments where quantity is >= 5 and <= 15.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE quantity BETWEEN 5 AND 15;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE quantity BETWEEN 5 AND 15;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c5a56e24a5878c81791a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gte", "question": "repairs ki entries lao jahan price 100 ya usse upar hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE price >= 100;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE price >= 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "66a886154d6a2a05a7f7_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "null", "question": "I need to check rows in repairs where name is null.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "95a0b0a0ed8d648db327_hinglish", "language": "hinglish", "backend": "supabase", "operation": "join_filter", "question": "Un rows ka enrollments.name lao jahan enrollments.group_id enrollments_groups.id se juda hai aur enrollments_groups.label South ke barabar hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT enrollments.name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT enrollments.name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0bdc9f428b0896d7e38a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sort_asc", "question": "retrieve all rows from enrollments and put them in order by balance lowest first", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments ORDER BY balance ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments ORDER BY balance ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4de9a85576f4fbfac7d2_hi", "language": "hi", "backend": "supabase", "operation": "all", "question": "rentals से सभी पंक्तियाँ लाइए।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_rentals", "query": "SELECT * FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_rentals\",\"query\":\"SELECT * FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ebf0786e1ce12c15da6f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "null", "question": "I need to check rows in deliveries where title is null.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3d12ba589211e391881d_en", "language": "en", "backend": "supabase", "operation": "not_null", "question": "Filter repairs to show only rows where title is not null.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE title IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE title IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ee24cd456d1464d72bf4_en", "language": "en", "backend": "supabase", "operation": "write", "question": "Blow away all rows from repairs.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}}
{"id": "70b8594259c7b8e10305_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "null", "question": "I need to check rows in deliveries where title is null.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5b11e37f42eb06caba31_en", "language": "en", "backend": "supabase", "operation": "not_null", "question": "Filter deliveries to show only rows where description is not null.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2b0c6cddd8771d41fc37_en", "language": "en", "backend": "supabase", "operation": "count_eq", "question": "How many instances in repairs have department as Pune?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT COUNT(*) FROM repairs WHERE department = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT COUNT(*) FROM repairs WHERE department = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c077714e2e488a364925_hinglish", "language": "hinglish", "backend": "supabase", "operation": "between", "question": "enrollments se wo rows show karo jahan quantity >= 5 aur <= 15 hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE quantity BETWEEN 5 AND 15;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE quantity BETWEEN 5 AND 15;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9912a7207dd6e403b2d5_hi", "language": "hi", "backend": "supabase", "operation": "top", "question": "repairs में price के उच्चतम मानों वाली 3 पंक्तियाँ प्रदर्शित करें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs ORDER BY price DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs ORDER BY price DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fe5044da918260a617a9_en", "language": "en", "backend": "supabase", "operation": "group_sum", "question": "I want to see the total cost for each status in repairs.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT status, SUM(cost) FROM repairs GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT status, SUM(cost) FROM repairs GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fe5044da918260a617a9_hinglish", "language": "hinglish", "backend": "supabase", "operation": "group_sum", "question": "Mujhe repairs me har status ke liye cost ka total dikhana hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT status, SUM(cost) FROM repairs GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT status, SUM(cost) FROM repairs GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "60b3e2da11c7ba5f35b3_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "year", "question": "Select repairs rows where recorded_at is in 2024.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE EXTRACT(YEAR FROM recorded_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE EXTRACT(YEAR FROM recorded_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "219cbf4b72e00c5d19c1_en", "language": "en", "backend": "supabase", "operation": "lt", "question": "Present the data from repairs where quantity is below 500.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE quantity < 500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE quantity < 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9ef4d4f5d5bcf5858087_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "and", "question": "give me rows from repairs where city is Electronics and price exceeds 10", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE city = 'Electronics' AND price > 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE city = 'Electronics' AND price > 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2a70571066edacbb19ac_hinglish", "language": "hinglish", "backend": "supabase", "operation": "bottom", "question": "rentals se score ke hisaab se 5 minimum rows, lowest value se shuru karte hue, display karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_rentals", "query": "SELECT * FROM rentals ORDER BY score ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_rentals\",\"query\":\"SELECT * FROM rentals ORDER BY score ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0676be3ca0df76c86a56_hi", "language": "hi", "backend": "supabase", "operation": "month", "question": "repairs में order_date 12 महीने में होने वाली पंक्तियां दिखाएं।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE EXTRACT(MONTH FROM order_date) = 12;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE EXTRACT(MONTH FROM order_date) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "03695bef6508dec8ebdb_en", "language": "en", "backend": "supabase", "operation": "join_filter", "question": "Select enrollments.customer_name from enrollments joined with enrollments_groups on enrollments.group_id where enrollments_groups.label is Delhi.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT enrollments.customer_name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT enrollments.customer_name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "03695bef6508dec8ebdb_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "join_filter", "question": "select enrollments.customer_name from enrollments joined to enrollments_groups on enrollments.group_id where enrollments_groups.label is Delhi please", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT enrollments.customer_name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT enrollments.customer_name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "69aa7071e087609fac06_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lte", "question": "repairs ka wo data dikhao jaha cost 10 se zyada nahi hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE cost <= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE cost <= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1140aefb82deeef59b85_en", "language": "en", "backend": "supabase", "operation": "sort_desc", "question": "Return all rows from repairs with balance in descending order.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs ORDER BY balance DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs ORDER BY balance DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1140aefb82deeef59b85_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sort_desc", "question": "repairs ke saare rows balance highest se lowest order mein return karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs ORDER BY balance DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs ORDER BY balance DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3800c9ec3cd53ff4c190_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "year", "question": "Select deliveries rows where recorded_at is in 2021.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE EXTRACT(YEAR FROM recorded_at) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5ec7a0dca8e2804dbca9_hi", "language": "hi", "backend": "supabase", "operation": "not_null", "question": "enrollments को फ़िल्टर करके सिर्फ वे पंक्तियाँ दिखाएं जहाँ item_name null नहीं है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE item_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE item_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "60b3e2da11c7ba5f35b3_en", "language": "en", "backend": "supabase", "operation": "year", "question": "Display repairs entries where recorded_at has the year 2024.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE EXTRACT(YEAR FROM recorded_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE EXTRACT(YEAR FROM recorded_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "de6c0f1e7617ba0be5cc_hi", "language": "hi", "backend": "supabase", "operation": "write", "question": "enrollments से हर एक पंक्ति को निकालें।", "expected": {"action": "clarify", "question": "यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "fc9a19b2881b35db88b1_en", "language": "en", "backend": "supabase", "operation": "gt", "question": "Select the rows from deliveries where score is greater than 500.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE score > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE score > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fc9a19b2881b35db88b1_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gt", "question": "look up the rows in deliveries where score is greater than 500", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE score > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE score > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "219cbf4b72e00c5d19c1_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lt", "question": "output the records from repairs where quantity is below 500 please.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE quantity < 500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE quantity < 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "848c830d91fb516d191b_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "not_null", "question": "Show the repairs rows where item_name is not null.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE item_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE item_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "97e851f9e56e234b3e99_en", "language": "en", "backend": "supabase", "operation": "project_eq", "question": "Output the display_name from deliveries where kind is active.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT display_name FROM deliveries WHERE kind = 'active';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT display_name FROM deliveries WHERE kind = 'active';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "97e851f9e56e234b3e99_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project_eq", "question": "show me the display_name in deliveries where kind is active", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT display_name FROM deliveries WHERE kind = 'active';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT display_name FROM deliveries WHERE kind = 'active';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "97e851f9e56e234b3e99_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project_eq", "question": "deliveries se display_name output karo jahan kind active hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT display_name FROM deliveries WHERE kind = 'active';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT display_name FROM deliveries WHERE kind = 'active';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3d12ba589211e391881d_hinglish", "language": "hinglish", "backend": "supabase", "operation": "not_null", "question": "repairs ko filter karke sirf wo rows dikhao jahan title null nahi hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE title IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE title IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "75b89b71dde15656327c_en", "language": "en", "backend": "supabase", "operation": "distinct", "question": "Pull out the distinct department values from the data in deliveries.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT DISTINCT department FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT DISTINCT department FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "75b89b71dde15656327c_hinglish", "language": "hinglish", "backend": "supabase", "operation": "distinct", "question": "deliveries mein maujood data se department ke distinct values pull karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT DISTINCT department FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT DISTINCT department FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a02ef9a534a2fa5110ea_hi", "language": "hi", "backend": "supabase", "operation": "and", "question": "deliveries से वे पंक्तियाँ प्रदर्शित करें जहाँ status Electronics है और salary 10000 से अधिक है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE status = 'Electronics' AND salary > 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE status = 'Electronics' AND salary > 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5b11e37f42eb06caba31_hinglish", "language": "hinglish", "backend": "supabase", "operation": "not_null", "question": "deliveries ko filter karke sirf wo rows dikhao jahan description null nahi hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2b0c6cddd8771d41fc37_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "count_eq", "question": "help me count rows in repairs where department is Pune", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT COUNT(*) FROM repairs WHERE department = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT COUNT(*) FROM repairs WHERE department = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2b0c6cddd8771d41fc37_hinglish", "language": "hinglish", "backend": "supabase", "operation": "count_eq", "question": "repairs mein department Pune wale data ki sankhya batayo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT COUNT(*) FROM repairs WHERE department = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT COUNT(*) FROM repairs WHERE department = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a888840d69a23410b604_en", "language": "en", "backend": "supabase", "operation": "top", "question": "Present the top 5 records from repairs by descending quantity value.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs ORDER BY quantity DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs ORDER BY quantity DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a888840d69a23410b604_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "top", "question": "Show the top 5 entries in repairs by quantity desc order.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs ORDER BY quantity DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs ORDER BY quantity DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a888840d69a23410b604_hinglish", "language": "hinglish", "backend": "supabase", "operation": "top", "question": "repairs mein quantity ke highest values wali 5 rows display karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs ORDER BY quantity DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs ORDER BY quantity DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "64405609b3558aed49bd_hi", "language": "hi", "backend": "supabase", "operation": "date_after", "question": "deliveries से वे रिकॉर्ड दिखाएं जहाँ order_date 2025-01-15 से पहले नहीं है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE order_date >= '2025-01-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE order_date >= '2025-01-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c077714e2e488a364925_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "between", "question": "show me the data from enrollments where quantity is between 5 and 15 inclusive of the boundaries", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE quantity BETWEEN 5 AND 15;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE quantity BETWEEN 5 AND 15;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3800c9ec3cd53ff4c190_en", "language": "en", "backend": "supabase", "operation": "year", "question": "Display deliveries entries where recorded_at has the year 2021.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE EXTRACT(YEAR FROM recorded_at) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9b0921e93df3d40ae856_hinglish", "language": "hinglish", "backend": "supabase", "operation": "bottom", "question": "enrollments se balance ke hisaab se 1 minimum rows, lowest value se shuru karte hue, display karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments ORDER BY balance ASC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments ORDER BY balance ASC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f9c2698db9e7dbe6ec8e_en", "language": "en", "backend": "supabase", "operation": "date_after", "question": "Query enrollments for rows where start_date is at or after 2025-09-15.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE start_date >= '2025-09-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE start_date >= '2025-09-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "60b3e2da11c7ba5f35b3_hinglish", "language": "hinglish", "backend": "supabase", "operation": "year", "question": "2024 ke recorded_at ke hisaab se repairs se data nikalo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE EXTRACT(YEAR FROM recorded_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE EXTRACT(YEAR FROM recorded_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8bacbbfb4da1879a2d23_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "list_tables", "question": "list out the tables in this database", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "fc9a19b2881b35db88b1_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gt", "question": "Mujhe deliveries se woh rows chahiye jahan score 500 se zyada hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE score > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE score > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9ef4d4f5d5bcf5858087_en", "language": "en", "backend": "supabase", "operation": "and", "question": "Select rows from repairs where city is Electronics and price is greater than 10.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE city = 'Electronics' AND price > 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE city = 'Electronics' AND price > 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dcc973b271bcbc227307_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "and", "question": "give me rows from repairs where region is North and balance exceeds 5", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE region = 'North' AND balance > 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE region = 'North' AND balance > 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "17da76d72f74206f6d3b_hinglish", "language": "hinglish", "backend": "supabase", "operation": "bottom", "question": "enrollments se salary ke hisaab se 20 minimum rows, lowest value se shuru karte hue, display karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments ORDER BY salary ASC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments ORDER BY salary ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3800c9ec3cd53ff4c190_hinglish", "language": "hinglish", "backend": "supabase", "operation": "year", "question": "2021 ke recorded_at ke hisaab se deliveries se data nikalo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE EXTRACT(YEAR FROM recorded_at) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bc6f054aa3be632d1bce_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "distinct", "question": "find the unique status in enrollments", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT DISTINCT status FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT DISTINCT status FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f9c2698db9e7dbe6ec8e_hinglish", "language": "hinglish", "backend": "supabase", "operation": "date_after", "question": "enrollments se wo records dikhao jahan start_date 2025-09-15 se pehle nahi hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE start_date >= '2025-09-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE start_date >= '2025-09-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8bacbbfb4da1879a2d23_en", "language": "en", "backend": "mysql", "operation": "list_tables", "question": "Fetch the table list for this database.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "8bacbbfb4da1879a2d23_hinglish", "language": "hinglish", "backend": "mysql", "operation": "list_tables", "question": "Kripya is database ki tables fetch karo.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "219cbf4b72e00c5d19c1_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lt", "question": "repairs se un rows ko show karo jahan quantity 500 se kam hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE quantity < 500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE quantity < 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "848c830d91fb516d191b_en", "language": "en", "backend": "supabase", "operation": "not_null", "question": "Filter repairs to show only rows where item_name is not null.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE item_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE item_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f638660fca2c9a9c9386_hi", "language": "hi", "backend": "supabase", "operation": "max", "question": "enrollments में salary की अधिकतम संख्या क्या है?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT MAX(salary) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT MAX(salary) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "03695bef6508dec8ebdb_hinglish", "language": "hinglish", "backend": "supabase", "operation": "join_filter", "question": "Un rows ka enrollments.customer_name lao jahan enrollments.group_id enrollments_groups.id se juda hai aur enrollments_groups.label Delhi ke barabar hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT enrollments.customer_name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT enrollments.customer_name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a1b7d1103527d4724102_en", "language": "en", "backend": "supabase", "operation": "describe", "question": "Fetch the metadata regarding columns and types for deliveries.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_deliveries", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "07beb228a10fcdc90b00_hi", "language": "hi", "backend": "mysql", "operation": "missing_schema", "question": "मेने डेटाबेस स्कीमा प्रदान नहीं की है, इसलिए enrollments के लिए SQL लिखने से पहले उपलब्ध तालिकाओं को खोजें।", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "c752dd15f0913fe63436_en", "language": "en", "backend": "supabase", "operation": "bottom", "question": "Pull the 5 rows with the lowest cost scores from repairs, smallest to largest.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs ORDER BY cost ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs ORDER BY cost ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6230fff878aa525542cd_hi", "language": "hi", "backend": "mysql", "operation": "missing_schema", "question": "मेने डेटाबेस स्कीमा प्रदान नहीं की है, इसलिए enrollments के लिए SQL लिखने से पहले उपलब्ध तालिकाओं को खोजें।", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "0a5c7e3d0753c8b54a5a_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "all", "question": "pull all rows from rentals", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "56594767710f98b9b347_en", "language": "en", "backend": "supabase", "operation": "between", "question": "Output the rows from repairs where rating is >= 5 and <= 15.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE rating BETWEEN 5 AND 15;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE rating BETWEEN 5 AND 15;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6bf08cbbd6b5cf97f755_en", "language": "en", "backend": "supabase", "operation": "sort_asc", "question": "Extract all rows from repairs and list them by cost in ascending order.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs ORDER BY cost ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs ORDER BY cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6bf08cbbd6b5cf97f755_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sort_asc", "question": "repairs ki saari rows cost ke kam se zyada order mein sort karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs ORDER BY cost ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs ORDER BY cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "619f53e951d6c4a5acdb_en", "language": "en", "backend": "supabase", "operation": "bottom", "question": "Pull the 1 rows with the lowest quantity scores from deliveries, smallest to largest.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries ORDER BY quantity ASC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries ORDER BY quantity ASC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9d71fdd5c658b26d6b5f_en", "language": "en", "backend": "supabase", "operation": "describe", "question": "Fetch the metadata regarding columns and types for repairs.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_repairs", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_repairs\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "3487e6b7bf3881dd7e68_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "max", "question": "get the maximum amount from enrollments please", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT MAX(amount) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT MAX(amount) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f9c2698db9e7dbe6ec8e_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "date_after", "question": "Show me enrollments data where start_date is not earlier than 2025-09-15.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE start_date >= '2025-09-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE start_date >= '2025-09-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "068560b98b4de1eee24e_hi", "language": "hi", "backend": "supabase", "operation": "contains", "question": "rentals में description में active रखने वाली पंक्तियाँ सूचीबद्ध करें, मामला-असंवेदनशीलता के साथ।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE LOWER(description) LIKE LOWER('%active%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE LOWER(description) LIKE LOWER('%active%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9ef4d4f5d5bcf5858087_hinglish", "language": "hinglish", "backend": "supabase", "operation": "and", "question": "repairs se wo rows display karo jahan city Electronics hai aur price 10 se zyada hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE city = 'Electronics' AND price > 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE city = 'Electronics' AND price > 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a5906e7b7aab0171e5db_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "avg", "question": "give me avg cost in rentals", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT AVG(cost) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT AVG(cost) FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dcc973b271bcbc227307_en", "language": "en", "backend": "supabase", "operation": "and", "question": "Select rows from repairs where region is North and balance is greater than 5.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE region = 'North' AND balance > 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE region = 'North' AND balance > 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "49dc5daf9217fe08b1c4_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sum", "question": "please find the total quantity in enrollments", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT SUM(quantity) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT SUM(quantity) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0a5c7e3d0753c8b54a5a_hinglish", "language": "hinglish", "backend": "mysql", "operation": "all", "question": "rentals se saare rows le aao.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "32af759abb6742f3aa01_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "all", "question": "pull all rows from enrollments", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "56594767710f98b9b347_hinglish", "language": "hinglish", "backend": "supabase", "operation": "between", "question": "repairs se wo rows show karo jahan rating >= 5 aur <= 15 hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE rating BETWEEN 5 AND 15;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE rating BETWEEN 5 AND 15;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f20e916f7b473213aa0d_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "and", "question": "give me rows from deliveries where kind is O'Reilly and rating exceeds 500", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE kind = 'O''Reilly' AND rating > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE kind = 'O''Reilly' AND rating > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e6b1d053d0a307dba955_hi", "language": "hi", "backend": "supabase", "operation": "or", "question": "rentals से department Books या North वाली पंक्तियाँ लें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_rentals", "query": "SELECT * FROM rentals WHERE department IN ('Books', 'North');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_rentals\",\"query\":\"SELECT * FROM rentals WHERE department IN ('Books', 'North');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d8445da89ec6e3d5a191_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "distinct", "question": "find the unique category in rentals", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT DISTINCT category FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT category FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6bf08cbbd6b5cf97f755_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sort_asc", "question": "retrieve all rows from repairs and put them in order by cost lowest first", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs ORDER BY cost ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs ORDER BY cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b18d2d8859825d11a5bf_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "avg", "question": "give me avg amount in rentals", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT AVG(amount) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT AVG(amount) FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "127c803e902f50fcb8e8_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "min", "question": "Show me minimum rating in enrollments.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT MIN(rating) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT MIN(rating) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3487e6b7bf3881dd7e68_en", "language": "en", "backend": "mysql", "operation": "max", "question": "Give me the highest amount for enrollments.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT MAX(amount) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT MAX(amount) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f0a44c49f89e568f5a97_en", "language": "en", "backend": "supabase", "operation": "join_filter", "question": "Select repairs.name from repairs joined with repairs_groups on repairs.group_id where repairs_groups.label is Delhi.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT repairs.name FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id WHERE repairs_groups.label = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT repairs.name FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id WHERE repairs_groups.label = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f0a44c49f89e568f5a97_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "join_filter", "question": "select repairs.name from repairs joined to repairs_groups on repairs.group_id where repairs_groups.label is Delhi please", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT repairs.name FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id WHERE repairs_groups.label = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT repairs.name FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id WHERE repairs_groups.label = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "494e7ce9785848f5299f_hi", "language": "hi", "backend": "supabase", "operation": "describe", "question": "rentals में मौजूद फील्ड्स और टाइप्स बताएं।", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_rentals", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_rentals\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "c3c086664aabec97406f_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "write", "question": "Clear enrollments completely.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "ebc657ab9bc6f3f33d21_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "having", "question": "Display department groups in repairs where row count is > 1000 and show the counts.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT department, COUNT(*) FROM repairs GROUP BY department HAVING COUNT(*) > 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT department, COUNT(*) FROM repairs GROUP BY department HAVING COUNT(*) > 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "848c830d91fb516d191b_hinglish", "language": "hinglish", "backend": "supabase", "operation": "not_null", "question": "repairs ko filter karke sirf wo rows dikhao jahan item_name null nahi hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE item_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE item_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a5906e7b7aab0171e5db_hinglish", "language": "hinglish", "backend": "mysql", "operation": "avg", "question": "rentals me cost ka average kya hai?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT AVG(cost) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT AVG(cost) FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8c437f27ac8f48915f9a_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "min", "question": "Show me minimum balance in rentals.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT MIN(balance) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT MIN(balance) FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6086ef96e9aab3a085d0_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "avg", "question": "give me avg balance in enrollments", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT AVG(balance) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT AVG(balance) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "41f7903ba296100a1de1_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "ambiguous", "question": "Show the leading records in rentals.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "0a5c7e3d0753c8b54a5a_en", "language": "en", "backend": "mysql", "operation": "all", "question": "Bring up all rows from rentals.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "780db517b686cfd2fce3_hi", "language": "hi", "backend": "supabase", "operation": "list_tables", "question": "कृपया इस डेटाबेस की तालिकाएँ fetch करें।", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_rentals", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_rentals\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "b13605951ead6efc54ba_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "describe", "question": "Display the field types and names for rentals.", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "rentals"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"rentals\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "b13605951ead6efc54ba_hinglish", "language": "hinglish", "backend": "mysql", "operation": "describe", "question": "Display the columns and data types for rentals.", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "rentals"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"rentals\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "94b5cc87d7b229fd2427_hi", "language": "hi", "backend": "mysql", "operation": "missing_schema", "question": "मेने डेटाबेस स्कीमा प्रदान नहीं की है, इसलिए deliveries के लिए SQL लिखने से पहले उपलब्ध तालिकाओं को खोजें।", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "b18d2d8859825d11a5bf_hinglish", "language": "hinglish", "backend": "mysql", "operation": "avg", "question": "rentals me amount ka average kya hai?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT AVG(amount) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT AVG(amount) FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "127c803e902f50fcb8e8_en", "language": "en", "backend": "mysql", "operation": "min", "question": "Give me the minimum rating from enrollments.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT MIN(rating) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT MIN(rating) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3487e6b7bf3881dd7e68_hinglish", "language": "hinglish", "backend": "mysql", "operation": "max", "question": "enrollments mein amount ki maximum number kya hai?", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT MAX(amount) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT MAX(amount) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e96284577cdc468f9024_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lte", "question": "select rentals rows where balance is not more than 20", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM rentals WHERE balance <= 20;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE balance <= 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6d9c843a92d3ec7fb99c_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_desc", "question": "show all rentals rows ordered by score highest to lowest", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM rentals ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM rentals ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ebc657ab9bc6f3f33d21_en", "language": "en", "backend": "supabase", "operation": "having", "question": "Identify the department groups in repairs with more than 1000 rows and report their respective counts.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT department, COUNT(*) FROM repairs GROUP BY department HAVING COUNT(*) > 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT department, COUNT(*) FROM repairs GROUP BY department HAVING COUNT(*) > 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ebc657ab9bc6f3f33d21_hinglish", "language": "hinglish", "backend": "supabase", "operation": "having", "question": "repairs se 1000 rows se zyada wale department groups ko unki row count ke saath dikhaiye.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT department, COUNT(*) FROM repairs GROUP BY department HAVING COUNT(*) > 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT department, COUNT(*) FROM repairs GROUP BY department HAVING COUNT(*) > 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a5906e7b7aab0171e5db_en", "language": "en", "backend": "mysql", "operation": "avg", "question": "What is the cost average in rentals?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT AVG(cost) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT AVG(cost) FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8c437f27ac8f48915f9a_en", "language": "en", "backend": "mysql", "operation": "min", "question": "Give me the minimum balance from rentals.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT MIN(balance) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT MIN(balance) FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8c437f27ac8f48915f9a_hinglish", "language": "hinglish", "backend": "mysql", "operation": "min", "question": "rentals mein balance ka minimum limit kya hai?", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT MIN(balance) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT MIN(balance) FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dcc973b271bcbc227307_hinglish", "language": "hinglish", "backend": "supabase", "operation": "and", "question": "repairs se wo rows display karo jahan region North hai aur balance 5 se zyada hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE region = 'North' AND balance > 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE region = 'North' AND balance > 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0e7660955234aff5d69a_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_sum", "question": "show the total salary for each category in rentals now", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT category, SUM(salary) FROM rentals GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT category, SUM(salary) FROM rentals GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "41f7903ba296100a1de1_hinglish", "language": "hinglish", "backend": "mysql", "operation": "ambiguous", "question": "rentals ke best records reveal karo.", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "0f9f5d9f2fda4132f1bf_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "contains", "question": "Bring rows from deliveries where display_name has Mumbai, no case distinction.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE LOWER(display_name) LIKE LOWER('%Mumbai%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE LOWER(display_name) LIKE LOWER('%Mumbai%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "32af759abb6742f3aa01_en", "language": "en", "backend": "mysql", "operation": "all", "question": "Bring up all rows from enrollments.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "32af759abb6742f3aa01_hinglish", "language": "hinglish", "backend": "mysql", "operation": "all", "question": "enrollments se saare rows le aao.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "56594767710f98b9b347_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "between", "question": "show me the data from repairs where rating is between 5 and 15 inclusive of the boundaries", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE rating BETWEEN 5 AND 15;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE rating BETWEEN 5 AND 15;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f20e916f7b473213aa0d_en", "language": "en", "backend": "supabase", "operation": "and", "question": "Select rows from deliveries where kind is O'Reilly and rating is greater than 500.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE kind = 'O''Reilly' AND rating > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE kind = 'O''Reilly' AND rating > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7b6982f2662ad49d01d8_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count", "question": "count the rows in enrollments", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e8d5bf3c1d106a2fb229_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_desc", "question": "show all rentals rows ordered by rating highest to lowest", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM rentals ORDER BY rating DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM rentals ORDER BY rating DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b18d2d8859825d11a5bf_en", "language": "en", "backend": "mysql", "operation": "avg", "question": "What is the amount average in rentals?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT AVG(amount) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT AVG(amount) FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "127c803e902f50fcb8e8_hinglish", "language": "hinglish", "backend": "mysql", "operation": "min", "question": "enrollments mein rating ka minimum limit kya hai?", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT MIN(rating) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT MIN(rating) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7dfb47d110d38ff48e06_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "not_null", "question": "Show the enrollments rows where title is not null.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM enrollments WHERE title IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments WHERE title IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f0a44c49f89e568f5a97_hinglish", "language": "hinglish", "backend": "supabase", "operation": "join_filter", "question": "Un rows ka repairs.name lao jahan repairs.group_id repairs_groups.id se juda hai aur repairs_groups.label Delhi ke barabar hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT repairs.name FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id WHERE repairs_groups.label = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT repairs.name FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id WHERE repairs_groups.label = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e8fe6a79395f381a5e01_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "ambiguous", "question": "Show the leading records in rentals.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "e96284577cdc468f9024_en", "language": "en", "backend": "mysql", "operation": "lte", "question": "Show the rentals data where balance is at most 20.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM rentals WHERE balance <= 20;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE balance <= 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a7342b948d7bd031d82c_hi", "language": "hi", "backend": "supabase", "operation": "min", "question": "deliveries में cost का न्यूनतम स्तर क्या है?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT MIN(cost) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT MIN(cost) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b9e6b86227c7663bd9bb_hi", "language": "hi", "backend": "supabase", "operation": "describe", "question": "rentals में मौजूद फील्ड्स और टाइप्स बताएं।", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_rentals", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_rentals\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "aca1abed450f2955c4ab_hi", "language": "hi", "backend": "mysql", "operation": "missing_schema", "question": "मेने डेटाबेस स्कीमा प्रदान नहीं की है, इसलिए deliveries के लिए SQL लिखने से पहले उपलब्ध तालिकाओं को खोजें।", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "9b02b7583735a1bdd0f2_hi", "language": "hi", "backend": "supabase", "operation": "ambiguous", "question": "enrollments के सर्वोत्तम रिकॉर्ड उजागर करें।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "8ecb63601a1ad54fe8ff_hi", "language": "hi", "backend": "supabase", "operation": "null", "question": "enrollments में name null होने वाली पंक्तियाँ लाएँ।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "64c0b80fc5c7f96a90b6_en", "language": "en", "backend": "supabase", "operation": "final", "question": "The database tool gave 109 rows. What is the count?", "expected": {"action": "answer", "text": "The tool returned 109 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 109 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "64c0b80fc5c7f96a90b6_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "final", "question": "the database tool gave 109 rows. how many rows is that?", "expected": {"action": "answer", "text": "The tool returned 109 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 109 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "49dc5daf9217fe08b1c4_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sum", "question": "enrollments mein quantity ka total measure karo.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT SUM(quantity) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT SUM(quantity) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6086ef96e9aab3a085d0_en", "language": "en", "backend": "mysql", "operation": "avg", "question": "What is the balance average in enrollments?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT AVG(balance) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT AVG(balance) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6086ef96e9aab3a085d0_hinglish", "language": "hinglish", "backend": "mysql", "operation": "avg", "question": "enrollments me balance ka average kya hai?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT AVG(balance) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT AVG(balance) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c752dd15f0913fe63436_hinglish", "language": "hinglish", "backend": "supabase", "operation": "bottom", "question": "repairs se cost ke hisaab se 5 minimum rows, lowest value se shuru karte hue, display karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs ORDER BY cost ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs ORDER BY cost ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4c61d39f7f226ce5244f_hi", "language": "hi", "backend": "supabase", "operation": "year", "question": "2025 के start_date के आधार पर enrollments से डेटा निकालें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE EXTRACT(YEAR FROM start_date) = 2025;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE EXTRACT(YEAR FROM start_date) = 2025;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "41f7903ba296100a1de1_en", "language": "en", "backend": "mysql", "operation": "ambiguous", "question": "Reveal the leading records in rentals.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "af4d35ab8d84f363ce1a_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project_eq", "question": "rentals se description output karo jahan status North hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT description FROM rentals WHERE status = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT description FROM rentals WHERE status = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "619f53e951d6c4a5acdb_hinglish", "language": "hinglish", "backend": "supabase", "operation": "bottom", "question": "deliveries se quantity ke hisaab se 1 minimum rows, lowest value se shuru karte hue, display karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries ORDER BY quantity ASC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries ORDER BY quantity ASC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c147e867055d7fd9f259_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "list_tables", "question": "list out the tables in this database", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "59fa7ad459e87198d29d_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "not_null", "question": "Show the enrollments rows where title is not null.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM enrollments WHERE title IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE title IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fe9e519448ad73aea083_hi", "language": "hi", "backend": "supabase", "operation": "list_tables", "question": "कृपया इस डेटाबेस की तालिकाएँ fetch करें।", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_deliveries", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "e8fe6a79395f381a5e01_hinglish", "language": "hinglish", "backend": "mysql", "operation": "ambiguous", "question": "rentals ke best records reveal karo.", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "e96284577cdc468f9024_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lte", "question": "rentals ka wo data dikhao jaha balance 20 se zyada nahi hai.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM rentals WHERE balance <= 20;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE balance <= 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "236309e4fd1627100f3a_hi", "language": "hi", "backend": "supabase", "operation": "sum", "question": "enrollments एंटीटी के लिए score का कुल आउटपुट दें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT SUM(score) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT SUM(score) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e956931ef5531d79954c_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "avg", "question": "give me avg salary in deliveries", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT AVG(salary) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT AVG(salary) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6d9c843a92d3ec7fb99c_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_desc", "question": "rentals ke saare rows score highest se lowest order mein return karo.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM rentals ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM rentals ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2ee0d715b288b4660747_hi", "language": "hi", "backend": "supabase", "operation": "contains", "question": "rentals में display_name में North रखने वाली पंक्तियाँ सूचीबद्ध करें, मामला-असंवेदनशीलता के साथ।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE LOWER(display_name) LIKE LOWER('%North%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE LOWER(display_name) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "46c2ec98a037d78a9576_hi", "language": "hi", "backend": "supabase", "operation": "list_tables", "question": "कृपया इस डेटाबेस की तालिकाएँ fetch करें।", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_deliveries", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "64c0b80fc5c7f96a90b6_hinglish", "language": "hinglish", "backend": "supabase", "operation": "final", "question": "Database tool ne 109 rows laayi hain. Kripya sankhya batao.", "expected": {"action": "answer", "text": "Tool ne 109 rows return ki."}, "output": "{\"action\":\"answer\",\"text\":\"Tool ne 109 rows return ki.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "49dc5daf9217fe08b1c4_en", "language": "en", "backend": "mysql", "operation": "sum", "question": "Measure the total quantity present in enrollments.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT SUM(quantity) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT SUM(quantity) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "21498b30adb6d60f9630_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "min", "question": "Show me minimum amount in enrollments.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT MIN(amount) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT MIN(amount) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ee24cd456d1464d72bf4_hi", "language": "hi", "backend": "supabase", "operation": "write", "question": "repairs से हर एक पंक्ति को निकालें।", "expected": {"action": "clarify", "question": "यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "fb320ffbb834f3c0a759_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gte", "question": "get rows from rentals where cost is 1 or above", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM rentals WHERE cost >= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE cost >= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3575acb51007208adc3f_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "not_null", "question": "Show the rentals rows where description is not null.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM rentals WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b6da31e0b488a9ecb9e4_en", "language": "en", "backend": "mysql", "operation": "count_eq", "question": "How many instances in rentals have region as Books?", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT COUNT(*) FROM rentals WHERE region = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM rentals WHERE region = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "84a09238c3eb1043c14c_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "avg", "question": "give me avg salary in repairs", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT AVG(salary) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT AVG(salary) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0f9f5d9f2fda4132f1bf_hinglish", "language": "hinglish", "backend": "supabase", "operation": "contains", "question": "Bring karo deliveries se un rows ko jinka display_name Mumbai hai, case difference ke bina.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE LOWER(display_name) LIKE LOWER('%Mumbai%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE LOWER(display_name) LIKE LOWER('%Mumbai%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "96a459251c82c550e710_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "write", "question": "Clear deliveries completely.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "af4d35ab8d84f363ce1a_en", "language": "en", "backend": "mysql", "operation": "project_eq", "question": "Output the description from rentals where status is North.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT description FROM rentals WHERE status = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT description FROM rentals WHERE status = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "af4d35ab8d84f363ce1a_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project_eq", "question": "show me the description in rentals where status is North", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT description FROM rentals WHERE status = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT description FROM rentals WHERE status = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f20e916f7b473213aa0d_hinglish", "language": "hinglish", "backend": "supabase", "operation": "and", "question": "deliveries se wo rows display karo jahan kind O'Reilly hai aur rating 500 se zyada hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE kind = 'O''Reilly' AND rating > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE kind = 'O''Reilly' AND rating > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e8d5bf3c1d106a2fb229_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_desc", "question": "rentals ke saare rows rating highest se lowest order mein return karo.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM rentals ORDER BY rating DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM rentals ORDER BY rating DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c147e867055d7fd9f259_en", "language": "en", "backend": "mysql", "operation": "list_tables", "question": "Fetch the table list for this database.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "c147e867055d7fd9f259_hinglish", "language": "hinglish", "backend": "mysql", "operation": "list_tables", "question": "Kripya is database ki tables fetch karo.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "eb4c2f1cc33712f0b82e_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "bottom", "question": "give me the 5 lowest cost data from enrollments ordered ascending", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM enrollments ORDER BY cost ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments ORDER BY cost ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2fae606ab30f319280e2_hinglish", "language": "hinglish", "backend": "mysql", "operation": "month", "question": "rentals mein order_date ka month 12 hone wali rows dikhao.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM rentals WHERE MONTH(order_date) = 12;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE MONTH(order_date) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4996e2672571480e6c60_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project", "question": "bring up customer_name from rentals only", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT customer_name FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT customer_name FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4996e2672571480e6c60_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project", "question": "i want just customer_name from rentals ok", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT customer_name FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT customer_name FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7dfb47d110d38ff48e06_en", "language": "en", "backend": "mysql", "operation": "not_null", "question": "Filter enrollments to show only rows where title is not null.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM enrollments WHERE title IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments WHERE title IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bc6f054aa3be632d1bce_en", "language": "en", "backend": "mysql", "operation": "distinct", "question": "Pull out the distinct status values from the data in enrollments.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT DISTINCT status FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT DISTINCT status FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bc6f054aa3be632d1bce_hinglish", "language": "hinglish", "backend": "mysql", "operation": "distinct", "question": "enrollments mein maujood data se status ke distinct values pull karo.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT DISTINCT status FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT DISTINCT status FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e8fe6a79395f381a5e01_en", "language": "en", "backend": "mysql", "operation": "ambiguous", "question": "Reveal the leading records in rentals.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "c3c086664aabec97406f_hinglish", "language": "hinglish", "backend": "mysql", "operation": "write", "question": "enrollments se har ek row remove karo.", "expected": {"action": "clarify", "question": "Yeh connection read-only hai. Kya SELECT query chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"Yeh connection read-only hai. Kya SELECT query chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "2d72fda52b91ee1560ab_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "all", "question": "pull all rows from deliveries", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6d9c843a92d3ec7fb99c_en", "language": "en", "backend": "mysql", "operation": "sort_desc", "question": "Return all rows from rentals with score in descending order.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM rentals ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM rentals ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6a9265d659c9399f1229_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_sum", "question": "show the total salary for each status in enrollments now", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT status, SUM(salary) FROM enrollments GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT status, SUM(salary) FROM enrollments GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f805b47d834fac71679e_hi", "language": "hi", "backend": "supabase", "operation": "join", "question": "rentals.group_id को rentals_groups.id से जोड़कर rentals.customer_name और rentals_groups.label का डेटा दिखाएँ।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT rentals.customer_name, rentals_groups.label FROM rentals JOIN rentals_groups ON rentals.group_id = rentals_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT rentals.customer_name, rentals_groups.label FROM rentals JOIN rentals_groups ON rentals.group_id = rentals_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "97e851f9e56e234b3e99_hi", "language": "hi", "backend": "supabase", "operation": "project_eq", "question": "deliveries से display_name आउटपुट करें जहाँ kind active है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT display_name FROM deliveries WHERE kind = 'active';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT display_name FROM deliveries WHERE kind = 'active';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dd9565e510782b49a5f0_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count", "question": "count the rows in deliveries", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT COUNT(*) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "21498b30adb6d60f9630_en", "language": "en", "backend": "mysql", "operation": "min", "question": "Give me the minimum amount from enrollments.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT MIN(amount) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT MIN(amount) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "01072ae077b62a8a9a43_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_desc", "question": "show all enrollments rows ordered by salary highest to lowest", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM enrollments ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c7aa166bcab44f298306_en", "language": "en", "backend": "mysql", "operation": "top", "question": "Present the top 1 records from enrollments by descending cost value.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM enrollments ORDER BY cost DESC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments ORDER BY cost DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c7aa166bcab44f298306_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "top", "question": "Show the top 1 entries in enrollments by cost desc order.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM enrollments ORDER BY cost DESC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments ORDER BY cost DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c7aa166bcab44f298306_hinglish", "language": "hinglish", "backend": "mysql", "operation": "top", "question": "enrollments mein cost ke highest values wali 1 rows display karo.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM enrollments ORDER BY cost DESC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments ORDER BY cost DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0e7660955234aff5d69a_en", "language": "en", "backend": "mysql", "operation": "group_sum", "question": "I want to see the total salary for each category in rentals.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT category, SUM(salary) FROM rentals GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT category, SUM(salary) FROM rentals GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0e7660955234aff5d69a_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_sum", "question": "Mujhe rentals me har category ke liye salary ka total dikhana hai.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT category, SUM(salary) FROM rentals GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT category, SUM(salary) FROM rentals GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8aeca0843f974e95bd97_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "list_tables", "question": "list out the tables in this database", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "0fe63ace7edb08819705_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project_eq", "question": "rentals se display_name output karo jahan kind Delhi hai.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT display_name FROM rentals WHERE kind = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT display_name FROM rentals WHERE kind = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fb320ffbb834f3c0a759_en", "language": "en", "backend": "mysql", "operation": "gte", "question": "Find records in rentals where cost is not below 1.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM rentals WHERE cost >= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE cost >= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fb320ffbb834f3c0a759_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gte", "question": "rentals ki entries lao jahan cost 1 ya usse upar hai.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM rentals WHERE cost >= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE cost >= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b6da31e0b488a9ecb9e4_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count_eq", "question": "rentals mein region Books wale data ki sankhya batayo.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT COUNT(*) FROM rentals WHERE region = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM rentals WHERE region = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "660dd90727fa904c6f30_hinglish", "language": "hinglish", "backend": "mysql", "operation": "top", "question": "rentals mein cost ke highest values wali 10 rows display karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM rentals ORDER BY cost DESC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM rentals ORDER BY cost DESC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1671c0540cd054060350_hi", "language": "hi", "backend": "supabase", "operation": "year", "question": "2022 के recorded_at के आधार पर enrollments से डेटा निकालें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE EXTRACT(YEAR FROM recorded_at) = 2022;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE EXTRACT(YEAR FROM recorded_at) = 2022;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2fbed66bad55619c4642_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "min", "question": "Show me minimum price in repairs.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT MIN(price) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT MIN(price) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d8445da89ec6e3d5a191_hinglish", "language": "hinglish", "backend": "mysql", "operation": "distinct", "question": "rentals mein maujood data se category ke distinct values pull karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT DISTINCT category FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT category FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e8d5bf3c1d106a2fb229_en", "language": "en", "backend": "mysql", "operation": "sort_desc", "question": "Return all rows from rentals with rating in descending order.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM rentals ORDER BY rating DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM rentals ORDER BY rating DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2c765382d84fd55136f4_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_count", "question": "rentals me har kind ke rows ka count do.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM rentals GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM rentals GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b0695b0692af5f9ff89e_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gte", "question": "get rows from enrollments where score is 10000 or above", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE score >= 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE score >= 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "59fa7ad459e87198d29d_en", "language": "en", "backend": "mysql", "operation": "not_null", "question": "Filter enrollments to show only rows where title is not null.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM enrollments WHERE title IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE title IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fc1a5aaf55bd30034ad0_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count", "question": "count the rows in deliveries", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4217b243f42c7af06169_en", "language": "en", "backend": "mysql", "operation": "lt", "question": "Present the data from enrollments where cost is below 10.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM enrollments WHERE cost < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments WHERE cost < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "467e53c90a42cd9935a3_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lte", "question": "select rentals rows where price is not more than 5", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM rentals WHERE price <= 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE price <= 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6391074b92a18a2d8964_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project", "question": "bring up display_name from enrollments only", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT display_name FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT display_name FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6391074b92a18a2d8964_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project", "question": "i want just display_name from enrollments ok", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT display_name FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT display_name FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e956931ef5531d79954c_en", "language": "en", "backend": "mysql", "operation": "avg", "question": "What is the salary average in deliveries?", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT AVG(salary) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT AVG(salary) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e956931ef5531d79954c_hinglish", "language": "hinglish", "backend": "mysql", "operation": "avg", "question": "deliveries me salary ka average kya hai?", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT AVG(salary) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT AVG(salary) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3cfa2682031af21eedd8_hi", "language": "hi", "backend": "supabase", "operation": "project", "question": "मुझे deliveries का name चाहिए, बाकी कुछ नहीं।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT name FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT name FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a5906e7b7aab0171e5db_hi", "language": "hi", "backend": "mysql", "operation": "avg", "question": "rentals में cost का औसत क्या है?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT AVG(cost) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT AVG(cost) FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "21498b30adb6d60f9630_hinglish", "language": "hinglish", "backend": "mysql", "operation": "min", "question": "enrollments mein amount ka minimum limit kya hai?", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT MIN(amount) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT MIN(amount) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f0c710492ac60feed37b_hi", "language": "hi", "backend": "supabase", "operation": "group_count", "question": "enrollments में प्रत्येक region के पंक्ति काउंट दें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT region, COUNT(*) FROM enrollments GROUP BY region;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT region, COUNT(*) FROM enrollments GROUP BY region;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8aeca0843f974e95bd97_en", "language": "en", "backend": "mysql", "operation": "list_tables", "question": "Fetch the table list for this database.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "8aeca0843f974e95bd97_hinglish", "language": "hinglish", "backend": "mysql", "operation": "list_tables", "question": "Kripya is database ki tables fetch karo.", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "0fe63ace7edb08819705_en", "language": "en", "backend": "mysql", "operation": "project_eq", "question": "Output the display_name from rentals where kind is Delhi.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT display_name FROM rentals WHERE kind = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT display_name FROM rentals WHERE kind = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0fe63ace7edb08819705_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project_eq", "question": "show me the display_name in rentals where kind is Delhi", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT display_name FROM rentals WHERE kind = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT display_name FROM rentals WHERE kind = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9adcc0998325a7b1eba4_hi", "language": "hi", "backend": "supabase", "operation": "lte", "question": "rentals का वो डेटा दिखाएं जहां quantity 100 से अधिक नहीं है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_rentals", "query": "SELECT * FROM rentals WHERE quantity <= 100;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_rentals\",\"query\":\"SELECT * FROM rentals WHERE quantity <= 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3575acb51007208adc3f_en", "language": "en", "backend": "mysql", "operation": "not_null", "question": "Filter rentals to show only rows where description is not null.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM rentals WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b6da31e0b488a9ecb9e4_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count_eq", "question": "help me count rows in rentals where region is Books", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT COUNT(*) FROM rentals WHERE region = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM rentals WHERE region = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "84a09238c3eb1043c14c_en", "language": "en", "backend": "mysql", "operation": "avg", "question": "What is the salary average in repairs?", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT AVG(salary) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT AVG(salary) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "84a09238c3eb1043c14c_hinglish", "language": "hinglish", "backend": "mysql", "operation": "avg", "question": "repairs me salary ka average kya hai?", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT AVG(salary) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT AVG(salary) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "660dd90727fa904c6f30_en", "language": "en", "backend": "mysql", "operation": "top", "question": "Present the top 10 records from rentals by descending cost value.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM rentals ORDER BY cost DESC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM rentals ORDER BY cost DESC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "660dd90727fa904c6f30_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "top", "question": "Show the top 10 entries in rentals by cost desc order.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM rentals ORDER BY cost DESC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM rentals ORDER BY cost DESC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b13605951ead6efc54ba_en", "language": "en", "backend": "mysql", "operation": "describe", "question": "Fetch the metadata regarding columns and types for rentals.", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "rentals"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"rentals\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a07055860746011ca946_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project", "question": "bring up display_name from enrollments only", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT display_name FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a07055860746011ca946_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project", "question": "i want just display_name from enrollments ok", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT display_name FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7b6982f2662ad49d01d8_en", "language": "en", "backend": "mysql", "operation": "count", "question": "List the number of rows found in enrollments.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2fbed66bad55619c4642_en", "language": "en", "backend": "mysql", "operation": "min", "question": "Give me the minimum price from repairs.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT MIN(price) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT MIN(price) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d8445da89ec6e3d5a191_en", "language": "en", "backend": "mysql", "operation": "distinct", "question": "Pull out the distinct category values from the data in rentals.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT DISTINCT category FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT category FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2c765382d84fd55136f4_en", "language": "en", "backend": "mysql", "operation": "group_count", "question": "Sum up the rows for each kind in rentals.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM rentals GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT kind, SUM(score) FROM rentals GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "2c765382d84fd55136f4_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_count", "question": "i need the total rows for each kind in rentals", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM rentals GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT kind, SUM(score) FROM rentals GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "62e8127db1df87f17dfc_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "write", "question": "Clear deliveries completely.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "f216d01e1724e4ca0e48_hi", "language": "hi", "backend": "supabase", "operation": "project", "question": "मुझे deliveries का name चाहिए, बाकी कुछ नहीं।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT name FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT name FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b0695b0692af5f9ff89e_en", "language": "en", "backend": "mysql", "operation": "gte", "question": "Find records in enrollments where score is not below 10000.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE score >= 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE score >= 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b18d2d8859825d11a5bf_hi", "language": "hi", "backend": "mysql", "operation": "avg", "question": "rentals में amount का औसत क्या है?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT AVG(amount) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT AVG(amount) FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2fae606ab30f319280e2_en", "language": "en", "backend": "mysql", "operation": "month", "question": "Provide rows from rentals where order_date's month value is 12.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM rentals WHERE MONTH(order_date) = 12;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE MONTH(order_date) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2fae606ab30f319280e2_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "month", "question": "give me the rows in rentals where month of order_date is 12", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM rentals WHERE MONTH(order_date) = 12;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE MONTH(order_date) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4217b243f42c7af06169_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lt", "question": "output the records from enrollments where cost is below 10 please.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM enrollments WHERE cost < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments WHERE cost < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "467e53c90a42cd9935a3_en", "language": "en", "backend": "mysql", "operation": "lte", "question": "Show the rentals data where price is at most 5.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM rentals WHERE price <= 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE price <= 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7dfb47d110d38ff48e06_hinglish", "language": "hinglish", "backend": "mysql", "operation": "not_null", "question": "enrollments ko filter karke sirf wo rows dikhao jahan title null nahi hai.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM enrollments WHERE title IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments WHERE title IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "33d0ab3caa0fba0e3985_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count", "question": "count the rows in repairs", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT COUNT(*) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dc2e72d9a61704c6679c_en", "language": "en", "backend": "mysql", "operation": "top", "question": "Present the top 10 records from enrollments by descending quantity value.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM enrollments ORDER BY quantity DESC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments ORDER BY quantity DESC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dc2e72d9a61704c6679c_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "top", "question": "Show the top 10 entries in enrollments by quantity desc order.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM enrollments ORDER BY quantity DESC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments ORDER BY quantity DESC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dc2e72d9a61704c6679c_hinglish", "language": "hinglish", "backend": "mysql", "operation": "top", "question": "enrollments mein quantity ke highest values wali 10 rows display karo.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM enrollments ORDER BY quantity DESC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments ORDER BY quantity DESC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c3c086664aabec97406f_en", "language": "en", "backend": "mysql", "operation": "write", "question": "Blow away all rows from enrollments.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}}
{"id": "2d72fda52b91ee1560ab_en", "language": "en", "backend": "mysql", "operation": "all", "question": "Bring up all rows from deliveries.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2d72fda52b91ee1560ab_hinglish", "language": "hinglish", "backend": "mysql", "operation": "all", "question": "deliveries se saare rows le aao.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a89baf1622de706da189_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "and", "question": "give me rows from rentals where status is Pune and amount exceeds 100", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE status = 'Pune' AND amount > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE status = 'Pune' AND amount > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c3d3a3d5475ef70e2f2c_en", "language": "en", "backend": "mysql", "operation": "sort_asc", "question": "Extract all rows from enrollments and list them by price in ascending order.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM enrollments ORDER BY price ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments ORDER BY price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c3d3a3d5475ef70e2f2c_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_asc", "question": "enrollments ki saari rows price ke kam se zyada order mein sort karo.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM enrollments ORDER BY price ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments ORDER BY price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fcd691cda0d5bb67b608_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "missing_value", "question": "filter rentals by city, i have not specified a value to use", "expected": {"action": "clarify", "question": "Which value should city equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should city equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "342e459fcb5dd23dad52_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "all", "question": "pull all rows from deliveries", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9955ba81eb264ff07981_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "ambiguous", "question": "Show the leading records in deliveries.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "eac1735177916d6d370c_hi", "language": "hi", "backend": "supabase", "operation": "join", "question": "enrollments.group_id को enrollments_groups.id से जोड़कर enrollments.display_name और enrollments_groups.label का डेटा दिखाएँ।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT enrollments.display_name, enrollments_groups.label FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT enrollments.display_name, enrollments_groups.label FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "32989df4383378821b6c_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "describe", "question": "Display the field types and names for deliveries.", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "deliveries"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"deliveries\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "32989df4383378821b6c_hinglish", "language": "hinglish", "backend": "mysql", "operation": "describe", "question": "Display the columns and data types for deliveries.", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "deliveries"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"deliveries\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "3575acb51007208adc3f_hinglish", "language": "hinglish", "backend": "mysql", "operation": "not_null", "question": "rentals ko filter karke sirf wo rows dikhao jahan description null nahi hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM rentals WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0f9f5d9f2fda4132f1bf_en", "language": "en", "backend": "supabase", "operation": "contains", "question": "Bring up the records in deliveries where display_name features Mumbai, without regard to case.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE LOWER(display_name) LIKE LOWER('%Mumbai%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE LOWER(display_name) LIKE LOWER('%Mumbai%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "21c860f0b85143a5adf6_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "max", "question": "get the maximum rating from deliveries please", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT MAX(rating) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT MAX(rating) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2fbed66bad55619c4642_hinglish", "language": "hinglish", "backend": "mysql", "operation": "min", "question": "repairs mein price ka minimum limit kya hai?", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT MIN(price) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT MIN(price) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1e54bf2ef5c5da89e926_hi", "language": "hi", "backend": "supabase", "operation": "having", "question": "enrollments से 1500 पंक्तियों से अधिक वाले category समूहों को उनकी पंक्ति संख्या के साथ दिखाइए।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM enrollments GROUP BY category HAVING COUNT(*) > 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM enrollments GROUP BY category HAVING COUNT(*) > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a2072435473a505c6fff_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "all", "question": "pull all rows from repairs", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "16b8ae3402d446c16d79_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project_eq", "question": "rentals se display_name output karo jahan kind South hai.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT display_name FROM rentals WHERE kind = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT display_name FROM rentals WHERE kind = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b0695b0692af5f9ff89e_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gte", "question": "enrollments ki entries lao jahan score 10000 ya usse upar hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE score >= 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE score >= 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "59fa7ad459e87198d29d_hinglish", "language": "hinglish", "backend": "mysql", "operation": "not_null", "question": "enrollments ko filter karke sirf wo rows dikhao jahan title null nahi hai.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM enrollments WHERE title IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE title IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "316b23c5d7f686804e72_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sum", "question": "please find the total score in repairs", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT SUM(score) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT SUM(score) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "af4683485d1360b8afb9_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "max", "question": "get the maximum quantity from repairs please", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT MAX(quantity) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT MAX(quantity) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e3314881b15c1b561692_hi", "language": "hi", "backend": "supabase", "operation": "having", "question": "rentals से 10000 पंक्तियों से अधिक वाले region समूहों को उनकी पंक्ति संख्या के साथ दिखाइए।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT region, COUNT(*) FROM rentals GROUP BY region HAVING COUNT(*) > 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM rentals GROUP BY region HAVING COUNT(*) > 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4996e2672571480e6c60_en", "language": "en", "backend": "mysql", "operation": "project", "question": "Let's look at customer_name in rentals specifically.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT customer_name FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT customer_name FROM rentals WHERE department = 'Aetnai';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "467e53c90a42cd9935a3_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lte", "question": "rentals ka wo data dikhao jaha price 5 se zyada nahi hai.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM rentals WHERE price <= 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE price <= 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dae8c9c830c197ee8e74_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_desc", "question": "show all repairs rows ordered by salary highest to lowest", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM repairs ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM repairs ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "66a886154d6a2a05a7f7_hi", "language": "hi", "backend": "supabase", "operation": "null", "question": "repairs में name null होने वाली पंक्तियाँ लाएँ।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c3d3a3d5475ef70e2f2c_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_asc", "question": "retrieve all rows from enrollments and put them in order by price lowest first", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM enrollments ORDER BY price ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments ORDER BY price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "de15f8143029f00970f6_hi", "language": "hi", "backend": "supabase", "operation": "bottom", "question": "deliveries से price के आधार पर 1 न्यूनतम पंक्तियां, सबसे कम मान से शुरू होकर, प्रदर्शित करें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries ORDER BY price ASC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries ORDER BY price ASC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "90eda6200c21cebc9077_en", "language": "en", "backend": "mysql", "operation": "group_count", "question": "Sum up the rows for each status in deliveries.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT status, COUNT(*) FROM deliveries GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT status, COUNT(*) FROM deliveries GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "90eda6200c21cebc9077_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_count", "question": "i need the total rows for each status in deliveries", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT status, COUNT(*) FROM deliveries GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT status, COUNT(*) FROM deliveries GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "90eda6200c21cebc9077_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_count", "question": "deliveries me har status ke rows ka count do.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT status, COUNT(*) FROM deliveries GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT status, COUNT(*) FROM deliveries GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9992776e0fbabf4da80e_hinglish", "language": "hinglish", "backend": "mysql", "operation": "month", "question": "rentals mein created_at ka month 12 hone wali rows dikhao.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM rentals WHERE MONTH(created_at) = 12;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE MONTH(created_at) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "502d4fde824d62a8f495_en", "language": "en", "backend": "supabase", "operation": "final", "question": "The database tool gave 13 rows. What is the count?", "expected": {"action": "answer", "text": "The tool returned 13 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 13 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "502d4fde824d62a8f495_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "final", "question": "the database tool gave 13 rows. how many rows is that?", "expected": {"action": "answer", "text": "The tool returned 13 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 13 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "6a9265d659c9399f1229_en", "language": "en", "backend": "mysql", "operation": "group_sum", "question": "I want to see the total salary for each status in enrollments.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT status, SUM(salary) FROM enrollments GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT status, SUM(salary) FROM enrollments GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6a9265d659c9399f1229_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_sum", "question": "Mujhe enrollments me har status ke liye salary ka total dikhana hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT status, SUM(salary) FROM enrollments GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT status, SUM(salary) FROM enrollments GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f661788465b66e8a5822_en", "language": "en", "backend": "mysql", "operation": "lt", "question": "Present the data from enrollments where score is below 200.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE score < 200;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE score < 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ebf0786e1ce12c15da6f_hi", "language": "hi", "backend": "supabase", "operation": "null", "question": "deliveries में title null होने वाली पंक्तियाँ लाएँ।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6086ef96e9aab3a085d0_hi", "language": "hi", "backend": "mysql", "operation": "avg", "question": "enrollments में balance का औसत क्या है?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT AVG(balance) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT AVG(balance) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "01072ae077b62a8a9a43_en", "language": "en", "backend": "mysql", "operation": "sort_desc", "question": "Return all rows from enrollments with salary in descending order.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM enrollments ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "01072ae077b62a8a9a43_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_desc", "question": "enrollments ke saare rows salary highest se lowest order mein return karo.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM enrollments ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "eab3dede73f7980dd2da_en", "language": "en", "backend": "mysql", "operation": "eq", "question": "Query enrollments to show all rows where department is Mumbai.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM enrollments WHERE department = 'Mumbai';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE department = 'Mumbai';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "eab3dede73f7980dd2da_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "eq", "question": "Show all rows in enrollments where department is Mumbai ok?", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM enrollments WHERE department = 'Mumbai';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE department = 'Mumbai';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "96a459251c82c550e710_hinglish", "language": "hinglish", "backend": "mysql", "operation": "write", "question": "deliveries se har ek row remove karo.", "expected": {"action": "clarify", "question": "Yeh connection read-only hai. Kya SELECT query chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"Yeh connection read-only hai. Kya SELECT query chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "70b8594259c7b8e10305_hi", "language": "hi", "backend": "supabase", "operation": "null", "question": "deliveries में title null होने वाली पंक्तियाँ लाएँ।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7b6982f2662ad49d01d8_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count", "question": "enrollments ki rows ka volume kya hai?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "21c860f0b85143a5adf6_en", "language": "en", "backend": "mysql", "operation": "max", "question": "Give me the highest rating for deliveries.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT MAX(rating) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT MAX(rating) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c49bd7e05e6964389dd5_hi", "language": "hi", "backend": "supabase", "operation": "ambiguous", "question": "deliveries के सर्वोत्तम रिकॉर्ड उजागर करें।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "16b8ae3402d446c16d79_en", "language": "en", "backend": "mysql", "operation": "project_eq", "question": "Output the display_name from rentals where kind is South.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT display_name FROM rentals WHERE kind = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT display_name FROM rentals WHERE kind = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "16b8ae3402d446c16d79_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project_eq", "question": "show me the display_name in rentals where kind is South", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT display_name FROM rentals WHERE kind = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT display_name FROM rentals WHERE kind = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "af4683485d1360b8afb9_en", "language": "en", "backend": "mysql", "operation": "max", "question": "Give me the highest quantity for repairs.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT MAX(quantity) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT MAX(quantity) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4217b243f42c7af06169_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lt", "question": "enrollments se un rows ko show karo jahan cost 10 se kam hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM enrollments WHERE cost < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments WHERE cost < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bc42ac714674eb7e9508_hi", "language": "hi", "backend": "supabase", "operation": "sum", "question": "repairs एंटीटी के लिए quantity का कुल आउटपुट दें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT SUM(quantity) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT SUM(quantity) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "640719c646d2ad1c92e6_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_asc", "question": "rentals ki saari rows rating ke kam se zyada order mein sort karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM rentals ORDER BY rating ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM rentals ORDER BY rating ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6391074b92a18a2d8964_en", "language": "en", "backend": "mysql", "operation": "project", "question": "Let's look at display_name in enrollments specifically.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT display_name FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT display_name FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a89baf1622de706da189_en", "language": "en", "backend": "mysql", "operation": "and", "question": "Select rows from rentals where status is Pune and amount is greater than 100.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE status = 'Pune' AND amount > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE status = 'Pune' AND amount > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "502d4fde824d62a8f495_hinglish", "language": "hinglish", "backend": "supabase", "operation": "final", "question": "Database tool ne 13 rows laayi hain. Kripya sankhya batao.", "expected": {"action": "answer", "text": "Tool ne 13 rows return ki."}, "output": "{\"action\":\"answer\",\"text\":\"Tool ne 13 rows return ki.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "342e459fcb5dd23dad52_en", "language": "en", "backend": "mysql", "operation": "all", "question": "Bring up all rows from deliveries.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "342e459fcb5dd23dad52_hinglish", "language": "hinglish", "backend": "mysql", "operation": "all", "question": "deliveries se saare rows le aao.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9955ba81eb264ff07981_en", "language": "en", "backend": "mysql", "operation": "ambiguous", "question": "Reveal the leading records in deliveries.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "9955ba81eb264ff07981_hinglish", "language": "hinglish", "backend": "mysql", "operation": "ambiguous", "question": "deliveries ke best records reveal karo.", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "f661788465b66e8a5822_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lt", "question": "output the records from enrollments where score is below 200 please.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE score < 200;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE score < 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dd9565e510782b49a5f0_en", "language": "en", "backend": "mysql", "operation": "count", "question": "List the number of rows found in deliveries.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT COUNT(*) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3f2bcb817dc0ac7fb510_en", "language": "en", "backend": "mysql", "operation": "sort_asc", "question": "Extract all rows from enrollments and list them by score in ascending order.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments ORDER BY score ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments ORDER BY score ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3f2bcb817dc0ac7fb510_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_asc", "question": "enrollments ki saari rows score ke kam se zyada order mein sort karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments ORDER BY score ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments ORDER BY score ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3920dafebc98270ffed5_en", "language": "en", "backend": "mysql", "operation": "between", "question": "Output the rows from enrollments where score is >= 500 and <= 1000.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM enrollments WHERE score BETWEEN 500 AND 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE score BETWEEN 500 AND 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b8b9810be462344512fe_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "min", "question": "Show me minimum score in deliveries.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT MIN(score) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(score) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a07055860746011ca946_en", "language": "en", "backend": "mysql", "operation": "project", "question": "Let's look at display_name in enrollments specifically.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT display_name FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "78e1c795f40293f04ae5_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sum", "question": "please find the total quantity in deliveries", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT SUM(quantity) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT SUM(quantity) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "21c860f0b85143a5adf6_hinglish", "language": "hinglish", "backend": "mysql", "operation": "max", "question": "deliveries mein rating ki maximum number kya hai?", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT MAX(rating) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT MAX(rating) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a2072435473a505c6fff_en", "language": "en", "backend": "mysql", "operation": "all", "question": "Bring up all rows from repairs.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a2072435473a505c6fff_hinglish", "language": "hinglish", "backend": "mysql", "operation": "all", "question": "repairs se saare rows le aao.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fc1a5aaf55bd30034ad0_en", "language": "en", "backend": "mysql", "operation": "count", "question": "List the number of rows found in deliveries.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "af4683485d1360b8afb9_hinglish", "language": "hinglish", "backend": "mysql", "operation": "max", "question": "repairs mein quantity ki maximum number kya hai?", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT MAX(quantity) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT MAX(quantity) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "eb4c2f1cc33712f0b82e_en", "language": "en", "backend": "mysql", "operation": "bottom", "question": "Pull the 5 rows with the lowest cost scores from enrollments, smallest to largest.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM enrollments ORDER BY cost ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments ORDER BY cost ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "640719c646d2ad1c92e6_en", "language": "en", "backend": "mysql", "operation": "sort_asc", "question": "Extract all rows from rentals and list them by rating in ascending order.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM rentals ORDER BY rating ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM rentals ORDER BY rating ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f2d0b2d997d6372721b0_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "missing_value", "question": "filter enrollments by department, i have not specified a value to use", "expected": {"action": "clarify", "question": "Which value should department equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should department equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "a89baf1622de706da189_hinglish", "language": "hinglish", "backend": "mysql", "operation": "and", "question": "rentals se wo rows display karo jahan status Pune hai aur amount 100 se zyada hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE status = 'Pune' AND amount > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE status = 'Pune' AND amount > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "60b3e2da11c7ba5f35b3_hi", "language": "hi", "backend": "supabase", "operation": "year", "question": "2024 के recorded_at के आधार पर repairs से डेटा निकालें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE EXTRACT(YEAR FROM recorded_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE EXTRACT(YEAR FROM recorded_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9992776e0fbabf4da80e_en", "language": "en", "backend": "mysql", "operation": "month", "question": "Provide rows from rentals where created_at's month value is 12.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM rentals WHERE MONTH(created_at) = 12;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE MONTH(created_at) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9992776e0fbabf4da80e_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "month", "question": "give me the rows in rentals where month of created_at is 12", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM rentals WHERE MONTH(created_at) = 12;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE MONTH(created_at) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "403211bcb94202245467_en", "language": "en", "backend": "mysql", "operation": "project_eq", "question": "Output the title from repairs where city is active.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT title FROM repairs WHERE city = 'active';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT title FROM repairs WHERE city = 'active';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "403211bcb94202245467_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project_eq", "question": "show me the title in repairs where city is active", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT title FROM repairs WHERE city = 'active';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT title FROM repairs WHERE city = 'active';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "403211bcb94202245467_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project_eq", "question": "repairs se title output karo jahan city active hai.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT title FROM repairs WHERE city = 'active';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT title FROM repairs WHERE city = 'active';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2e9ccfbc4c276796c15f_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lte", "question": "select deliveries rows where balance is not more than 200", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE balance <= 200;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE balance <= 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "96006b107903118566cc_en", "language": "en", "backend": "mysql", "operation": "group_count", "question": "Sum up the rows for each kind in deliveries.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT kind, COUNT(*) FROM deliveries GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT kind, COUNT(*) FROM deliveries GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "96006b107903118566cc_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_count", "question": "i need the total rows for each kind in deliveries", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT kind, COUNT(*) FROM deliveries GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT kind, COUNT(*) FROM deliveries GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "96006b107903118566cc_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_count", "question": "deliveries me har kind ke rows ka count do.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT kind, COUNT(*) FROM deliveries GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT kind, COUNT(*) FROM deliveries GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a1b7d1103527d4724102_hi", "language": "hi", "backend": "supabase", "operation": "describe", "question": "deliveries में मौजूद फील्ड्स और टाइप्स बताएं।", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_deliveries", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "9f4ae4d3b48c49a8c943_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gte", "question": "get rows from deliveries where salary is 2 or above", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM deliveries WHERE salary >= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries WHERE salary >= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3f2bcb817dc0ac7fb510_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_asc", "question": "retrieve all rows from enrollments and put them in order by score lowest first", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments ORDER BY score ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments ORDER BY score ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0a5c7e3d0753c8b54a5a_hi", "language": "hi", "backend": "mysql", "operation": "all", "question": "rentals से सभी पंक्तियाँ लाइए।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3920dafebc98270ffed5_hinglish", "language": "hinglish", "backend": "mysql", "operation": "between", "question": "enrollments se wo rows show karo jahan score >= 500 aur <= 1000 hai.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM enrollments WHERE score BETWEEN 500 AND 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE score BETWEEN 500 AND 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b8b9810be462344512fe_en", "language": "en", "backend": "mysql", "operation": "min", "question": "Give me the minimum score from deliveries.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT MIN(score) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(score) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "68d0c440e1210d5f5590_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_sum", "question": "show the total score for each kind in repairs now", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT kind, SUM(score) FROM repairs GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT kind, SUM(score) FROM repairs GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "96a459251c82c550e710_en", "language": "en", "backend": "mysql", "operation": "write", "question": "Blow away all rows from deliveries.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}}
{"id": "9b4ac5b2bb0340639986_hinglish", "language": "hinglish", "backend": "mysql", "operation": "month", "question": "enrollments mein order_date ka month 6 hone wali rows dikhao.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE MONTH(order_date) = 6;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE MONTH(order_date) = 6;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9d71fdd5c658b26d6b5f_hi", "language": "hi", "backend": "supabase", "operation": "describe", "question": "repairs में मौजूद फील्ड्स और टाइप्स बताएं।", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_repairs", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_repairs\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "62e8127db1df87f17dfc_hinglish", "language": "hinglish", "backend": "mysql", "operation": "write", "question": "deliveries se har ek row remove karo.", "expected": {"action": "clarify", "question": "Yeh connection read-only hai. Kya SELECT query chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"Yeh connection read-only hai. Kya SELECT query chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "316b23c5d7f686804e72_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sum", "question": "repairs mein score ka total measure karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT SUM(score) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT SUM(score) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3800c9ec3cd53ff4c190_hi", "language": "hi", "backend": "supabase", "operation": "year", "question": "2021 के recorded_at के आधार पर deliveries से डेटा निकालें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE EXTRACT(YEAR FROM recorded_at) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "33d0ab3caa0fba0e3985_en", "language": "en", "backend": "mysql", "operation": "count", "question": "List the number of rows found in repairs.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT COUNT(*) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dae8c9c830c197ee8e74_en", "language": "en", "backend": "mysql", "operation": "sort_desc", "question": "Return all rows from repairs with salary in descending order.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM repairs ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM repairs ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dae8c9c830c197ee8e74_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_desc", "question": "repairs ke saare rows salary highest se lowest order mein return karo.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM repairs ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM repairs ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "640719c646d2ad1c92e6_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_asc", "question": "retrieve all rows from rentals and put them in order by rating lowest first", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM rentals ORDER BY rating ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM rentals ORDER BY rating ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ccbdc5b0dcc9fcc00608_hi", "language": "hi", "backend": "supabase", "operation": "month", "question": "rentals में recorded_at 10 महीने में होने वाली पंक्तियां दिखाएं।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_rentals", "query": "SELECT * FROM rentals WHERE EXTRACT(MONTH FROM recorded_at) = 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_rentals\",\"query\":\"SELECT * FROM rentals WHERE EXTRACT(MONTH FROM recorded_at) = 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e956931ef5531d79954c_hi", "language": "hi", "backend": "mysql", "operation": "avg", "question": "deliveries में salary का औसत क्या है?", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT AVG(salary) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT AVG(salary) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fcd691cda0d5bb67b608_en", "language": "en", "backend": "mysql", "operation": "missing_value", "question": "Please filter the records in rentals by city, but I have not specified a value.", "expected": {"action": "clarify", "question": "Which value should city equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should city equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "219cbf4b72e00c5d19c1_hi", "language": "hi", "backend": "supabase", "operation": "lt", "question": "repairs में वे पंक्तियाँ खोजें जहाँ quantity 500 से कम है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE quantity < 500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE quantity < 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2e9ccfbc4c276796c15f_en", "language": "en", "backend": "mysql", "operation": "lte", "question": "Show the deliveries data where balance is at most 200.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE balance <= 200;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE balance <= 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4cb74692efd909e4f3fd_hi", "language": "hi", "backend": "supabase", "operation": "count", "question": "enrollments में स्टोर की गई पंक्तियों की संख्या क्या है?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT COUNT(*) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT COUNT(*) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3b0a2346d3ae970242c1_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "year", "question": "Select deliveries rows where recorded_at is in 2023.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM deliveries WHERE YEAR(recorded_at) = 2023;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries WHERE YEAR(recorded_at) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f661788465b66e8a5822_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lt", "question": "enrollments se un rows ko show karo jahan score 200 se kam hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE score < 200;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE score < 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9f4ae4d3b48c49a8c943_en", "language": "en", "backend": "mysql", "operation": "gte", "question": "Find records in deliveries where salary is not below 2.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM deliveries WHERE salary >= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries WHERE salary >= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "69aa7071e087609fac06_hi", "language": "hi", "backend": "supabase", "operation": "lte", "question": "repairs का वो डेटा दिखाएं जहां cost 10 से अधिक नहीं है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE cost <= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE cost <= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2d039adc19e8ba87f706_en", "language": "en", "backend": "mysql", "operation": "or", "question": "Select rows from deliveries where category is Pune or North.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM deliveries WHERE category IN ('Pune', 'North');"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE category IN ('Pune', 'North');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2d039adc19e8ba87f706_hinglish", "language": "hinglish", "backend": "mysql", "operation": "or", "question": "deliveries se category Pune ya North wali rows le lo.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM deliveries WHERE category IN ('Pune', 'North');"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE category IN ('Pune', 'North');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dd9565e510782b49a5f0_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count", "question": "deliveries ki rows ka volume kya hai?", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT COUNT(*) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "b9d861b71317ad86ce3e_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "having", "question": "Display category groups in enrollments where row count is > 2 and show the counts.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM enrollments GROUP BY category HAVING COUNT(*) > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM enrollments GROUP BY category HAVING COUNT(*) > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8d03186d0c3faa663273_hinglish", "language": "hinglish", "backend": "mysql", "operation": "month", "question": "deliveries mein order_date ka month 1 hone wali rows dikhao.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM deliveries WHERE MONTH(order_date) = 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE MONTH(order_date) = 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "eab3dede73f7980dd2da_hinglish", "language": "hinglish", "backend": "mysql", "operation": "eq", "question": "enrollments se query karo aur wo saari rows dikhao jahan department Mumbai hai.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM enrollments WHERE department = 'Mumbai';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE department = 'Mumbai';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "40a4c97e6c006d30b593_hi", "language": "hi", "backend": "supabase", "operation": "count", "question": "enrollments में स्टोर की गई पंक्तियों की संख्या क्या है?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT COUNT(*) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT COUNT(*) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "84a09238c3eb1043c14c_hi", "language": "hi", "backend": "mysql", "operation": "avg", "question": "repairs में salary का औसत क्या है?", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT AVG(salary) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT AVG(salary) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b8b9810be462344512fe_hinglish", "language": "hinglish", "backend": "mysql", "operation": "min", "question": "deliveries mein score ka minimum limit kya hai?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT MIN(score) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(score) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "af4d35ab8d84f363ce1a_hi", "language": "hi", "backend": "mysql", "operation": "project_eq", "question": "rentals से description आउटपुट करें जहाँ status North है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT description FROM rentals WHERE status = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT description FROM rentals WHERE status = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9b4ac5b2bb0340639986_en", "language": "en", "backend": "mysql", "operation": "month", "question": "Provide rows from enrollments where order_date's month value is 6.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE MONTH(order_date) = 6;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE MONTH(order_date) = 6;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9b4ac5b2bb0340639986_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "month", "question": "give me the rows in enrollments where month of order_date is 6", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE MONTH(order_date) = 6;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE MONTH(order_date) = 6;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4a26377ced6478bb8d0e_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "and", "question": "give me rows from repairs where city is Electronics and quantity exceeds 200", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM repairs WHERE city = 'Electronics' AND quantity > 200;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM repairs WHERE city = 'Electronics' AND quantity > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fc1a5aaf55bd30034ad0_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count", "question": "deliveries ki rows ka volume kya hai?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "316b23c5d7f686804e72_en", "language": "en", "backend": "mysql", "operation": "sum", "question": "Measure the total score present in repairs.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT SUM(score) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT SUM(score) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "13ac9d307d6b101b8705_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gte", "question": "get rows from repairs where cost is 0 or above", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs WHERE cost >= 0;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE cost >= 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dafbcd52face40a7b678_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_sum", "question": "show the total rating for each kind in repairs now", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT kind, SUM(rating) FROM repairs GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT kind, SUM(rating) FROM repairs GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f551f3aa4afc6f91a2d5_hi", "language": "hi", "backend": "supabase", "operation": "having", "question": "repairs से 100 पंक्तियों से अधिक वाले category समूहों को उनकी पंक्ति संख्या के साथ दिखाइए।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7839648878bf9feda17b_en", "language": "en", "backend": "mysql", "operation": "count_eq", "question": "How many instances in repairs have department as Pune?", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT COUNT(*) FROM repairs WHERE department = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM repairs WHERE department = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8926f167d28d5c555478_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gte", "question": "get rows from deliveries where quantity is 50 or above", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM deliveries WHERE quantity >= 50;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE quantity >= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e213d3002d5c7836cc0a_en", "language": "en", "backend": "mysql", "operation": "between", "question": "Output the rows from repairs where amount is >= 1000 and <= 1100.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM repairs WHERE amount BETWEEN 1000 AND 1100;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE amount BETWEEN 1000 AND 1100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dbc778be5f23038bcc3d_en", "language": "en", "backend": "mysql", "operation": "count_eq", "question": "How many instances in repairs have category as O'Reilly?", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT COUNT(*) FROM repairs WHERE category = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM repairs WHERE category = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3b0a2346d3ae970242c1_en", "language": "en", "backend": "mysql", "operation": "year", "question": "Display deliveries entries where recorded_at has the year 2023.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM deliveries WHERE YEAR(recorded_at) = 2023;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries WHERE YEAR(recorded_at) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6569be8d680d87bb283d_en", "language": "en", "backend": "mysql", "operation": "gt", "question": "Select the rows from deliveries where amount is greater than 200.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM deliveries WHERE amount > 200;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries WHERE amount > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6569be8d680d87bb283d_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gt", "question": "look up the rows in deliveries where amount is greater than 200", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM deliveries WHERE amount > 200;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries WHERE amount > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9f4ae4d3b48c49a8c943_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gte", "question": "deliveries ki entries lao jahan salary 2 ya usse upar hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM deliveries WHERE salary >= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries WHERE salary >= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b9d861b71317ad86ce3e_en", "language": "en", "backend": "mysql", "operation": "having", "question": "Identify the category groups in enrollments with more than 2 rows and report their respective counts.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM enrollments GROUP BY category HAVING COUNT(*) > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM enrollments GROUP BY category HAVING COUNT(*) > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b9d861b71317ad86ce3e_hinglish", "language": "hinglish", "backend": "mysql", "operation": "having", "question": "enrollments se 2 rows se zyada wale category groups ko unki row count ke saath dikhaiye.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM enrollments GROUP BY category HAVING COUNT(*) > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM enrollments GROUP BY category HAVING COUNT(*) > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0e015b584f6a6179890c_en", "language": "en", "backend": "mysql", "operation": "date_after", "question": "Query rentals for rows where created_at is at or after 2023-05-15.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM rentals WHERE created_at >= '2023-05-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE created_at >= '2023-05-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0e015b584f6a6179890c_hinglish", "language": "hinglish", "backend": "mysql", "operation": "date_after", "question": "rentals se wo records dikhao jahan created_at 2023-05-15 se pehle nahi hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM rentals WHERE created_at >= '2023-05-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE created_at >= '2023-05-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8d03186d0c3faa663273_en", "language": "en", "backend": "mysql", "operation": "month", "question": "Provide rows from deliveries where order_date's month value is 1.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM deliveries WHERE MONTH(order_date) = 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE MONTH(order_date) = 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8d03186d0c3faa663273_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "month", "question": "give me the rows in deliveries where month of order_date is 1", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM deliveries WHERE MONTH(order_date) = 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE MONTH(order_date) = 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "32989df4383378821b6c_en", "language": "en", "backend": "mysql", "operation": "describe", "question": "Fetch the metadata regarding columns and types for deliveries.", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "deliveries"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"deliveries\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "3920dafebc98270ffed5_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "between", "question": "show me the data from enrollments where score is between 500 and 1000 inclusive of the boundaries", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM enrollments WHERE score BETWEEN 500 AND 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE score BETWEEN 500 AND 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "32af759abb6742f3aa01_hi", "language": "hi", "backend": "mysql", "operation": "all", "question": "enrollments से सभी पंक्तियाँ लाइए।", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e309fa17d1bc158b8c8f_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lte", "question": "select deliveries rows where quantity is not more than 5", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE quantity <= 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE quantity <= 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "78e1c795f40293f04ae5_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sum", "question": "deliveries mein quantity ka total measure karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT SUM(quantity) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT SUM(quantity) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "cd2773edfc61c2b4bc9a_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_sum", "question": "show the total rating for each department in deliveries now", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT department, SUM(rating) FROM deliveries GROUP BY department;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT department, SUM(rating) FROM deliveries GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "62e8127db1df87f17dfc_en", "language": "en", "backend": "mysql", "operation": "write", "question": "Blow away all rows from deliveries.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}}
{"id": "e053da65d9fd664996e6_en", "language": "en", "backend": "mysql", "operation": "gt", "question": "Select the rows from repairs where amount is greater than 20.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM repairs WHERE amount > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM repairs WHERE amount > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e053da65d9fd664996e6_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gt", "question": "look up the rows in repairs where amount is greater than 20", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM repairs WHERE amount > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM repairs WHERE amount > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c077714e2e488a364925_hi", "language": "hi", "backend": "supabase", "operation": "between", "question": "enrollments से वे पंक्तियाँ प्रदर्शित करें जहाँ quantity >= 5 और <= 15 है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE quantity BETWEEN 5 AND 15;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE quantity BETWEEN 5 AND 15;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0e9353d09ce2f0cd9994_en", "language": "en", "backend": "mysql", "operation": "count_eq", "question": "How many instances in deliveries have city as O'Reilly?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT COUNT(*) FROM deliveries WHERE city = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM deliveries WHERE city = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "13ac9d307d6b101b8705_en", "language": "en", "backend": "mysql", "operation": "gte", "question": "Find records in repairs where cost is not below 0.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs WHERE cost >= 0;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE cost >= 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1667f375dfed0ad934a4_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "and", "question": "give me rows from repairs where city is Books and rating exceeds 1500", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs WHERE city = 'Books' AND rating > 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE city = 'Books' AND rating > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "328b77dd66cdf8cabfd8_en", "language": "en", "backend": "mysql", "operation": "or", "question": "Select rows from repairs where status is O'Reilly or South.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs WHERE status IN ('O''Reilly', 'South');"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE status IN ('O''Reilly', 'South');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "328b77dd66cdf8cabfd8_hinglish", "language": "hinglish", "backend": "mysql", "operation": "or", "question": "repairs se status O'Reilly ya South wali rows le lo.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs WHERE status IN ('O''Reilly', 'South');"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE status IN ('O''Reilly', 'South');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "33d0ab3caa0fba0e3985_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count", "question": "repairs ki rows ka volume kya hai?", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT COUNT(*) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "029cd7c058965ed14b05_en", "language": "en", "backend": "mysql", "operation": "eq", "question": "Query repairs to show all rows where department is active.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs WHERE department = 'active';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE department = 'active';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "029cd7c058965ed14b05_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "eq", "question": "Show all rows in repairs where department is active ok?", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs WHERE department = 'active';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE department = 'active';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e27bab397ec7daf2a101_hi", "language": "hi", "backend": "supabase", "operation": "distinct", "question": "enrollments में मौजूद डेटा से kind के विशिष्ट मान निकालें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT DISTINCT kind FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT DISTINCT kind FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2daba35926021b3697eb_en", "language": "en", "backend": "mysql", "operation": "date_after", "question": "Query enrollments for rows where recorded_at is at or after 2023-05-15.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE recorded_at >= '2023-05-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE recorded_at >= '2023-05-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8926f167d28d5c555478_en", "language": "en", "backend": "mysql", "operation": "gte", "question": "Find records in deliveries where quantity is not below 50.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM deliveries WHERE quantity >= 50;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE quantity >= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2e9ccfbc4c276796c15f_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lte", "question": "deliveries ka wo data dikhao jaha balance 200 se zyada nahi hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE balance <= 200;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE balance <= 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e213d3002d5c7836cc0a_hinglish", "language": "hinglish", "backend": "mysql", "operation": "between", "question": "repairs se wo rows show karo jahan amount >= 1000 aur <= 1100 hai.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM repairs WHERE amount BETWEEN 1000 AND 1100;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE amount BETWEEN 1000 AND 1100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8c437f27ac8f48915f9a_hi", "language": "hi", "backend": "mysql", "operation": "min", "question": "rentals में balance का न्यूनतम स्तर क्या है?", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT MIN(balance) FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT MIN(balance) FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3b0a2346d3ae970242c1_hinglish", "language": "hinglish", "backend": "mysql", "operation": "year", "question": "2023 ke recorded_at ke hisaab se deliveries se data nikalo.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM deliveries WHERE YEAR(recorded_at) = 2023;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries WHERE YEAR(recorded_at) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6569be8d680d87bb283d_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gt", "question": "Mujhe deliveries se woh rows chahiye jahan amount 200 se zyada hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM deliveries WHERE amount > 200;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries WHERE amount > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2d039adc19e8ba87f706_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "or", "question": "Please show rows from deliveries where category is either Pune or North.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM deliveries WHERE category IN ('Pune', 'North');"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE category IN ('Pune', 'North');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b93dfa01b85949af4005_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "contains", "question": "Bring rows from deliveries where description has Mumbai, no case distinction.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM deliveries WHERE LOWER(description) LIKE LOWER('%Mumbai%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE LOWER(description) LIKE LOWER('%Mumbai%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c2f17e8891f4e861494c_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "missing_value", "question": "filter repairs by department, i have not specified a value to use", "expected": {"action": "clarify", "question": "Which value should department equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should department equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "0fe63ace7edb08819705_hi", "language": "hi", "backend": "mysql", "operation": "project_eq", "question": "rentals से display_name आउटपुट करें जहाँ kind Delhi है।", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT display_name FROM rentals WHERE kind = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT display_name FROM rentals WHERE kind = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f637ccd41642ab190d04_hinglish", "language": "hinglish", "backend": "mysql", "operation": "null", "question": "repairs mein customer_name null hone wali rows lao.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs WHERE customer_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE customer_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "68d0c440e1210d5f5590_en", "language": "en", "backend": "mysql", "operation": "group_sum", "question": "I want to see the total score for each kind in repairs.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT kind, SUM(score) FROM repairs GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT kind, SUM(score) FROM repairs GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "68d0c440e1210d5f5590_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_sum", "question": "Mujhe repairs me har kind ke liye score ka total dikhana hai.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT kind, SUM(score) FROM repairs GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT kind, SUM(score) FROM repairs GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8cb6fe8c4ec5ccf02866_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "missing_value", "question": "filter deliveries by city, i have not specified a value to use", "expected": {"action": "clarify", "question": "Which value should city equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should city equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "d9f1ee5cf1a15f1ad44b_en", "language": "en", "backend": "mysql", "operation": "eq", "question": "Query repairs to show all rows where department is Books.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM repairs WHERE department = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM repairs WHERE department = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d9f1ee5cf1a15f1ad44b_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "eq", "question": "Show all rows in repairs where department is Books ok?", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM repairs WHERE department = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM repairs WHERE department = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e309fa17d1bc158b8c8f_en", "language": "en", "backend": "mysql", "operation": "lte", "question": "Show the deliveries data where quantity is at most 5.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE quantity <= 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE quantity <= 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "78e1c795f40293f04ae5_en", "language": "en", "backend": "mysql", "operation": "sum", "question": "Measure the total quantity present in deliveries.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT SUM(quantity) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT SUM(quantity) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "04165cd31bcf23c6fac9_en", "language": "en", "backend": "mysql", "operation": "eq", "question": "Query repairs to show all rows where category is O'Reilly.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE category = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE category = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "04165cd31bcf23c6fac9_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "eq", "question": "Show all rows in repairs where category is O'Reilly ok?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE category = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE category = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e053da65d9fd664996e6_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gt", "question": "Mujhe repairs se woh rows chahiye jahan amount 20 se zyada hai.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM repairs WHERE amount > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM repairs WHERE amount > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4a26377ced6478bb8d0e_en", "language": "en", "backend": "mysql", "operation": "and", "question": "Select rows from repairs where city is Electronics and quantity is greater than 200.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM repairs WHERE city = 'Electronics' AND quantity > 200;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM repairs WHERE city = 'Electronics' AND quantity > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5cf994b39a1eb0a5d0b6_hinglish", "language": "hinglish", "backend": "mysql", "operation": "null", "question": "deliveries mein item_name null hone wali rows lao.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM deliveries WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "127c803e902f50fcb8e8_hi", "language": "hi", "backend": "mysql", "operation": "min", "question": "enrollments में rating का न्यूनतम स्तर क्या है?", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT MIN(rating) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT MIN(rating) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "eb4c2f1cc33712f0b82e_hinglish", "language": "hinglish", "backend": "mysql", "operation": "bottom", "question": "enrollments se cost ke hisaab se 5 minimum rows, lowest value se shuru karte hue, display karo.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM enrollments ORDER BY cost ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments ORDER BY cost ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "13ac9d307d6b101b8705_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gte", "question": "repairs ki entries lao jahan cost 0 ya usse upar hai.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs WHERE cost >= 0;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE cost >= 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e0b7adca29f0eb32fe2c_en", "language": "en", "backend": "mysql", "operation": "between", "question": "Output the rows from deliveries where quantity is >= 3 and <= 53.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM deliveries WHERE quantity BETWEEN 3 AND 53;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE quantity BETWEEN 3 AND 53;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3487e6b7bf3881dd7e68_hi", "language": "hi", "backend": "mysql", "operation": "max", "question": "enrollments में amount की अधिकतम संख्या क्या है?", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT MAX(amount) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT MAX(amount) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f2d0b2d997d6372721b0_en", "language": "en", "backend": "mysql", "operation": "missing_value", "question": "Please filter the records in enrollments by department, but I have not specified a value.", "expected": {"action": "clarify", "question": "Which value should department equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should department equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "c5a56e24a5878c81791a_hi", "language": "hi", "backend": "supabase", "operation": "gte", "question": "repairs की वे एंट्री प्राप्त करें जहाँ price 100 या उससे ऊपर है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE price >= 100;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE price >= 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7839648878bf9feda17b_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count_eq", "question": "help me count rows in repairs where department is Pune", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT COUNT(*) FROM repairs WHERE department = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM repairs WHERE department = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7839648878bf9feda17b_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count_eq", "question": "repairs mein department Pune wale data ki sankhya batayo.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT COUNT(*) FROM repairs WHERE department = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM repairs WHERE department = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2daba35926021b3697eb_hinglish", "language": "hinglish", "backend": "mysql", "operation": "date_after", "question": "enrollments se wo records dikhao jahan recorded_at 2023-05-15 se pehle nahi hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE recorded_at >= '2023-05-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE recorded_at >= '2023-05-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8926f167d28d5c555478_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gte", "question": "deliveries ki entries lao jahan quantity 50 ya usse upar hai.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM deliveries WHERE quantity >= 50;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE quantity >= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dbc778be5f23038bcc3d_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count_eq", "question": "help me count rows in repairs where category is O'Reilly", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT COUNT(*) FROM repairs WHERE category = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM repairs WHERE category = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dbc778be5f23038bcc3d_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count_eq", "question": "repairs mein category O'Reilly wale data ki sankhya batayo.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT COUNT(*) FROM repairs WHERE category = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM repairs WHERE category = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5e4c3931848b0640bfaa_en", "language": "en", "backend": "mysql", "operation": "top", "question": "Present the top 10 records from deliveries by descending quantity value.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries ORDER BY quantity DESC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries ORDER BY quantity DESC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5e4c3931848b0640bfaa_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "top", "question": "Show the top 10 entries in deliveries by quantity desc order.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries ORDER BY quantity DESC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries ORDER BY quantity DESC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5e4c3931848b0640bfaa_hinglish", "language": "hinglish", "backend": "mysql", "operation": "top", "question": "deliveries mein quantity ke highest values wali 10 rows display karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries ORDER BY quantity DESC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries ORDER BY quantity DESC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0e015b584f6a6179890c_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "date_after", "question": "Show me rentals data where created_at is not earlier than 2023-05-15.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM rentals WHERE created_at >= '2023-05-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE created_at >= '2023-05-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "68ec6ad5b5ff4ebdc86a_en", "language": "en", "backend": "mysql", "operation": "sort_asc", "question": "Extract all rows from deliveries and list them by balance in ascending order.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries ORDER BY balance ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries ORDER BY balance ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "68ec6ad5b5ff4ebdc86a_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_asc", "question": "deliveries ki saari rows balance ke kam se zyada order mein sort karo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries ORDER BY balance ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries ORDER BY balance ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6aaf184e4cafcf2f5c16_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "having", "question": "Display category groups in repairs where row count is > 20 and show the counts.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "00e1fcbe466a7fdf0d8a_en", "language": "en", "backend": "mysql", "operation": "date_after", "question": "Query rentals for rows where order_date is at or after 2023-11-15.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM rentals WHERE order_date >= '2023-11-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE order_date >= '2023-11-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "00e1fcbe466a7fdf0d8a_hinglish", "language": "hinglish", "backend": "mysql", "operation": "date_after", "question": "rentals se wo records dikhao jahan order_date 2023-11-15 se pehle nahi hai.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM rentals WHERE order_date >= '2023-11-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE order_date >= '2023-11-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9325b9d4f4ee1c176f2f_en", "language": "en", "backend": "mysql", "operation": "gt", "question": "Select the rows from repairs where score is greater than 1500.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM repairs WHERE score > 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM repairs WHERE score > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9325b9d4f4ee1c176f2f_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gt", "question": "look up the rows in repairs where score is greater than 1500", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM repairs WHERE score > 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM repairs WHERE score > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "57e4f37da2636ee67319_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "contains", "question": "Bring rows from enrollments where item_name has active, no case distinction.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE LOWER(item_name) LIKE LOWER('%active%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE LOWER(item_name) LIKE LOWER('%active%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0e9353d09ce2f0cd9994_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count_eq", "question": "help me count rows in deliveries where city is O'Reilly", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT COUNT(*) FROM deliveries WHERE city = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM deliveries WHERE city = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0e9353d09ce2f0cd9994_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count_eq", "question": "deliveries mein city O'Reilly wale data ki sankhya batayo.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT COUNT(*) FROM deliveries WHERE city = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM deliveries WHERE city = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e0b7adca29f0eb32fe2c_hinglish", "language": "hinglish", "backend": "mysql", "operation": "between", "question": "deliveries se wo rows show karo jahan quantity >= 3 aur <= 53 hai.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM deliveries WHERE quantity BETWEEN 3 AND 53;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE quantity BETWEEN 3 AND 53;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1667f375dfed0ad934a4_en", "language": "en", "backend": "mysql", "operation": "and", "question": "Select rows from repairs where city is Books and rating is greater than 1500.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs WHERE city = 'Books' AND rating > 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE city = 'Books' AND rating > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "328b77dd66cdf8cabfd8_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "or", "question": "Please show rows from repairs where status is either O'Reilly or South.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs WHERE status IN ('O''Reilly', 'South');"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE status IN ('O''Reilly', 'South');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dafbcd52face40a7b678_en", "language": "en", "backend": "mysql", "operation": "group_sum", "question": "I want to see the total rating for each kind in repairs.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT kind, SUM(rating) FROM repairs GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT kind, SUM(rating) FROM repairs GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dafbcd52face40a7b678_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_sum", "question": "Mujhe repairs me har kind ke liye rating ka total dikhana hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT kind, SUM(rating) FROM repairs GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT kind, SUM(rating) FROM repairs GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2daba35926021b3697eb_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "date_after", "question": "Show me enrollments data where recorded_at is not earlier than 2023-05-15.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE recorded_at >= '2023-05-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE recorded_at >= '2023-05-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fcd691cda0d5bb67b608_hinglish", "language": "hinglish", "backend": "mysql", "operation": "missing_value", "question": "Please rentals mein records ko city ke hisaab se filter karo, lekin maine koi value specify nahi ki hai.", "expected": {"action": "clarify", "question": "city ki kaunsi value chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"city ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "e213d3002d5c7836cc0a_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "between", "question": "show me the data from repairs where amount is between 1000 and 1100 inclusive of the boundaries", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM repairs WHERE amount BETWEEN 1000 AND 1100;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE amount BETWEEN 1000 AND 1100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b93dfa01b85949af4005_hinglish", "language": "hinglish", "backend": "mysql", "operation": "contains", "question": "Bring karo deliveries se un rows ko jinka description Mumbai hai, case difference ke bina.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM deliveries WHERE LOWER(description) LIKE LOWER('%Mumbai%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE LOWER(description) LIKE LOWER('%Mumbai%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f637ccd41642ab190d04_en", "language": "en", "backend": "mysql", "operation": "null", "question": "Show me the rows from repairs where customer_name is null.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs WHERE customer_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE customer_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "68ec6ad5b5ff4ebdc86a_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_asc", "question": "retrieve all rows from deliveries and put them in order by balance lowest first", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries ORDER BY balance ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries ORDER BY balance ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6aaf184e4cafcf2f5c16_en", "language": "en", "backend": "mysql", "operation": "having", "question": "Identify the category groups in repairs with more than 20 rows and report their respective counts.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6aaf184e4cafcf2f5c16_hinglish", "language": "hinglish", "backend": "mysql", "operation": "having", "question": "repairs se 20 rows se zyada wale category groups ko unki row count ke saath dikhaiye.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9325b9d4f4ee1c176f2f_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gt", "question": "Mujhe repairs se woh rows chahiye jahan score 1500 se zyada hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM repairs WHERE score > 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM repairs WHERE score > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e309fa17d1bc158b8c8f_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lte", "question": "deliveries ka wo data dikhao jaha quantity 5 se zyada nahi hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE quantity <= 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE quantity <= 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "cd2773edfc61c2b4bc9a_en", "language": "en", "backend": "mysql", "operation": "group_sum", "question": "I want to see the total rating for each department in deliveries.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT department, SUM(rating) FROM deliveries GROUP BY department;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT department, SUM(rating) FROM deliveries GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "cd2773edfc61c2b4bc9a_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_sum", "question": "Mujhe deliveries me har department ke liye rating ka total dikhana hai.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT department, SUM(rating) FROM deliveries GROUP BY department;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT department, SUM(rating) FROM deliveries GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7942659b264a287de6f0_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "missing_value", "question": "filter repairs by department, i have not specified a value to use", "expected": {"action": "clarify", "question": "Which value should department equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should department equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "16b8ae3402d446c16d79_hi", "language": "hi", "backend": "mysql", "operation": "project_eq", "question": "rentals से display_name आउटपुट करें जहाँ kind South है।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT display_name FROM rentals WHERE kind = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT display_name FROM rentals WHERE kind = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4a26377ced6478bb8d0e_hinglish", "language": "hinglish", "backend": "mysql", "operation": "and", "question": "repairs se wo rows display karo jahan city Electronics hai aur quantity 200 se zyada hai.", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM repairs WHERE city = 'Electronics' AND quantity > 200;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM repairs WHERE city = 'Electronics' AND quantity > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5cf994b39a1eb0a5d0b6_en", "language": "en", "backend": "mysql", "operation": "null", "question": "Show me the rows from deliveries where item_name is null.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM deliveries WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f9c2698db9e7dbe6ec8e_hi", "language": "hi", "backend": "supabase", "operation": "date_after", "question": "enrollments से वे रिकॉर्ड दिखाएं जहाँ start_date 2025-09-15 से पहले नहीं है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments WHERE start_date >= '2025-09-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments WHERE start_date >= '2025-09-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c3c086664aabec97406f_hi", "language": "hi", "backend": "mysql", "operation": "write", "question": "enrollments से हर एक पंक्ति को निकालें।", "expected": {"action": "clarify", "question": "यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "029cd7c058965ed14b05_hinglish", "language": "hinglish", "backend": "mysql", "operation": "eq", "question": "repairs se query karo aur wo saari rows dikhao jahan department active hai.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs WHERE department = 'active';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE department = 'active';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "384c3b042d68b5a13b59_en", "language": "en", "backend": "mysql", "operation": "between", "question": "Output the rows from deliveries where salary is >= 200 and <= 300.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM deliveries WHERE salary BETWEEN 200 AND 300;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE salary BETWEEN 200 AND 300;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8bacbbfb4da1879a2d23_hi", "language": "hi", "backend": "mysql", "operation": "list_tables", "question": "कृपया इस डेटाबेस की तालिकाएँ fetch करें।", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "c4a83c9634139691ddde_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "having", "question": "Display category groups in repairs where row count is > 200 and show the counts.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 200;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "202d7bd6d9a637a071e8_en", "language": "en", "backend": "mysql", "operation": "between", "question": "Output the rows from deliveries where cost is >= 50 and <= 60.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM deliveries WHERE cost BETWEEN 50 AND 60;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries WHERE cost BETWEEN 50 AND 60;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "41f7903ba296100a1de1_hi", "language": "hi", "backend": "mysql", "operation": "ambiguous", "question": "rentals के सर्वोत्तम रिकॉर्ड उजागर करें।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "f637ccd41642ab190d04_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "null", "question": "I need to check rows in repairs where customer_name is null.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs WHERE customer_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE customer_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "00e1fcbe466a7fdf0d8a_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "date_after", "question": "Show me rentals data where order_date is not earlier than 2023-11-15.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM rentals WHERE order_date >= '2023-11-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE order_date >= '2023-11-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b13605951ead6efc54ba_hi", "language": "hi", "backend": "mysql", "operation": "describe", "question": "rentals में मौजूद फील्ड्स और टाइप्स बताएं।", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "rentals"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"rentals\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "d9f1ee5cf1a15f1ad44b_hinglish", "language": "hinglish", "backend": "mysql", "operation": "eq", "question": "repairs se query karo aur wo saari rows dikhao jahan department Books hai.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM repairs WHERE department = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM repairs WHERE department = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "57e4f37da2636ee67319_hinglish", "language": "hinglish", "backend": "mysql", "operation": "contains", "question": "Bring karo enrollments se un rows ko jinka item_name active hai, case difference ke bina.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE LOWER(item_name) LIKE LOWER('%active%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE LOWER(item_name) LIKE LOWER('%active%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "04165cd31bcf23c6fac9_hinglish", "language": "hinglish", "backend": "mysql", "operation": "eq", "question": "repairs se query karo aur wo saari rows dikhao jahan category O'Reilly hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE category = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE category = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5cf994b39a1eb0a5d0b6_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "null", "question": "I need to check rows in deliveries where item_name is null.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM deliveries WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e0b7adca29f0eb32fe2c_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "between", "question": "show me the data from deliveries where quantity is between 3 and 53 inclusive of the boundaries", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM deliveries WHERE quantity BETWEEN 3 AND 53;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE quantity BETWEEN 3 AND 53;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1667f375dfed0ad934a4_hinglish", "language": "hinglish", "backend": "mysql", "operation": "and", "question": "repairs se wo rows display karo jahan city Books hai aur rating 1500 se zyada hai.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs WHERE city = 'Books' AND rating > 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE city = 'Books' AND rating > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2d72fda52b91ee1560ab_hi", "language": "hi", "backend": "mysql", "operation": "all", "question": "deliveries से सभी पंक्तियाँ लाइए।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "384c3b042d68b5a13b59_hinglish", "language": "hinglish", "backend": "mysql", "operation": "between", "question": "deliveries se wo rows show karo jahan salary >= 200 aur <= 300 hai.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM deliveries WHERE salary BETWEEN 200 AND 300;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE salary BETWEEN 200 AND 300;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fe5044da918260a617a9_hi", "language": "hi", "backend": "supabase", "operation": "group_sum", "question": "मुझे repairs में प्रत्येक status के लिए cost का कुल दिखाना है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT status, SUM(cost) FROM repairs GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT status, SUM(cost) FROM repairs GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b654c9f6e4c53883151c_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "join", "question": "retrieve repairs.description and repairs_groups.label join repairs.group_id to repairs_groups.id", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT repairs.description, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT repairs.description, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "95a0b0a0ed8d648db327_hi", "language": "hi", "backend": "supabase", "operation": "join_filter", "question": "उन्हीं रिकॉर्ड्स का enrollments.name लें जहाँ enrollments.group_id enrollments_groups.id से जुड़ा है और enrollments_groups.label South के बराबर है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT enrollments.name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT enrollments.name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0bdc9f428b0896d7e38a_hi", "language": "hi", "backend": "supabase", "operation": "sort_asc", "question": "enrollments की सभी पंक्तियाँ balance के कम से ज्यादा क्रम में क्रमबद्ध करें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments ORDER BY balance ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments ORDER BY balance ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c4a83c9634139691ddde_en", "language": "en", "backend": "mysql", "operation": "having", "question": "Identify the category groups in repairs with more than 200 rows and report their respective counts.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 200;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c4a83c9634139691ddde_hinglish", "language": "hinglish", "backend": "mysql", "operation": "having", "question": "repairs se 200 rows se zyada wale category groups ko unki row count ke saath dikhaiye.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 200;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "cd68dcf59fb97c84247d_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "contains", "question": "Bring rows from repairs where item_name has Mumbai, no case distinction.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM repairs WHERE LOWER(item_name) LIKE LOWER('%Mumbai%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE LOWER(item_name) LIKE LOWER('%Mumbai%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "202d7bd6d9a637a071e8_hinglish", "language": "hinglish", "backend": "mysql", "operation": "between", "question": "deliveries se wo rows show karo jahan cost >= 50 aur <= 60 hai.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM deliveries WHERE cost BETWEEN 50 AND 60;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries WHERE cost BETWEEN 50 AND 60;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "21498b30adb6d60f9630_hi", "language": "hi", "backend": "mysql", "operation": "min", "question": "enrollments में amount का न्यूनतम स्तर क्या है?", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT MIN(amount) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT MIN(amount) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c2f17e8891f4e861494c_en", "language": "en", "backend": "mysql", "operation": "missing_value", "question": "Please filter the records in repairs by department, but I have not specified a value.", "expected": {"action": "clarify", "question": "Which value should department equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should department equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "8cb6fe8c4ec5ccf02866_en", "language": "en", "backend": "mysql", "operation": "missing_value", "question": "Please filter the records in deliveries by city, but I have not specified a value.", "expected": {"action": "clarify", "question": "Which value should city equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should city equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "56594767710f98b9b347_hi", "language": "hi", "backend": "supabase", "operation": "between", "question": "repairs से वे पंक्तियाँ प्रदर्शित करें जहाँ rating >= 5 और <= 15 है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE rating BETWEEN 5 AND 15;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE rating BETWEEN 5 AND 15;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e8fe6a79395f381a5e01_hi", "language": "hi", "backend": "mysql", "operation": "ambiguous", "question": "rentals के सर्वोत्तम रिकॉर्ड उजागर करें।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "f2d0b2d997d6372721b0_hinglish", "language": "hinglish", "backend": "mysql", "operation": "missing_value", "question": "Please enrollments mein records ko department ke hisaab se filter karo, lekin maine koi value specify nahi ki hai.", "expected": {"action": "clarify", "question": "department ki kaunsi value chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"department ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "403211bcb94202245467_hi", "language": "hi", "backend": "mysql", "operation": "project_eq", "question": "repairs से title आउटपुट करें जहाँ city active है।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT title FROM repairs WHERE city = 'active';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT title FROM repairs WHERE city = 'active';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fc9a19b2881b35db88b1_hi", "language": "hi", "backend": "supabase", "operation": "gt", "question": "मुझे deliveries से वे पंक्तियाँ चाहिए जहाँ score 500 से अधिक है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE score > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE score > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "75b89b71dde15656327c_hi", "language": "hi", "backend": "supabase", "operation": "distinct", "question": "deliveries में मौजूद डेटा से department के विशिष्ट मान निकालें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT DISTINCT department FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT DISTINCT department FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b93dfa01b85949af4005_en", "language": "en", "backend": "mysql", "operation": "contains", "question": "Bring up the records in deliveries where description features Mumbai, without regard to case.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM deliveries WHERE LOWER(description) LIKE LOWER('%Mumbai%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE LOWER(description) LIKE LOWER('%Mumbai%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2fbed66bad55619c4642_hi", "language": "hi", "backend": "mysql", "operation": "min", "question": "repairs में price का न्यूनतम स्तर क्या है?", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT MIN(price) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT MIN(price) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e96284577cdc468f9024_hi", "language": "hi", "backend": "mysql", "operation": "lte", "question": "rentals का वो डेटा दिखाएं जहां balance 20 से अधिक नहीं है।", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM rentals WHERE balance <= 20;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE balance <= 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "384c3b042d68b5a13b59_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "between", "question": "show me the data from deliveries where salary is between 200 and 300 inclusive of the boundaries", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM deliveries WHERE salary BETWEEN 200 AND 300;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE salary BETWEEN 200 AND 300;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "342e459fcb5dd23dad52_hi", "language": "hi", "backend": "mysql", "operation": "all", "question": "deliveries से सभी पंक्तियाँ लाइए।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "cd68dcf59fb97c84247d_hinglish", "language": "hinglish", "backend": "mysql", "operation": "contains", "question": "Bring karo repairs se un rows ko jinka item_name Mumbai hai, case difference ke bina.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM repairs WHERE LOWER(item_name) LIKE LOWER('%Mumbai%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE LOWER(item_name) LIKE LOWER('%Mumbai%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "202d7bd6d9a637a071e8_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "between", "question": "show me the data from deliveries where cost is between 50 and 60 inclusive of the boundaries", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM deliveries WHERE cost BETWEEN 50 AND 60;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries WHERE cost BETWEEN 50 AND 60;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "49dc5daf9217fe08b1c4_hi", "language": "hi", "backend": "mysql", "operation": "sum", "question": "enrollments एंटीटी के लिए quantity का कुल आउटपुट दें।", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT SUM(quantity) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT SUM(quantity) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "96a459251c82c550e710_hi", "language": "hi", "backend": "mysql", "operation": "write", "question": "deliveries से हर एक पंक्ति को निकालें।", "expected": {"action": "clarify", "question": "यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "57e4f37da2636ee67319_en", "language": "en", "backend": "mysql", "operation": "contains", "question": "Bring up the records in enrollments where item_name features active, without regard to case.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE LOWER(item_name) LIKE LOWER('%active%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE LOWER(item_name) LIKE LOWER('%active%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7942659b264a287de6f0_en", "language": "en", "backend": "mysql", "operation": "missing_value", "question": "Please filter the records in repairs by department, but I have not specified a value.", "expected": {"action": "clarify", "question": "Which value should department equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should department equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "a2072435473a505c6fff_hi", "language": "hi", "backend": "mysql", "operation": "all", "question": "repairs से सभी पंक्तियाँ लाइए।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fed9927b6460e7bd91b2_hi", "language": "hi", "backend": "supabase", "operation": "sql_error", "question": "क्योंकि अंतिम क्वेरी अज्ञात कॉलम के कारण फेल हुई, repairs स्कीमा का निरीक्षण करने के बाद ही दोबारा प्रयास करें।", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"project_id": "demo_repairs", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"project_id\":\"demo_repairs\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "1140aefb82deeef59b85_hi", "language": "hi", "backend": "supabase", "operation": "sort_desc", "question": "repairs से सभी पंक्तियाँ balance के उच्चतम से निम्नतम क्रम में लौटाएँ।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs ORDER BY balance DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs ORDER BY balance DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "71a64ddc45ae6b8498d9_en", "language": "en", "backend": "mysql", "operation": "final", "question": "The database tool gave 13 rows. What is the count?", "expected": {"action": "answer", "text": "The tool returned 13 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 13 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "71a64ddc45ae6b8498d9_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "final", "question": "the database tool gave 13 rows. how many rows is that?", "expected": {"action": "answer", "text": "The tool returned 13 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 13 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "2b0c6cddd8771d41fc37_hi", "language": "hi", "backend": "supabase", "operation": "count_eq", "question": "repairs में department Pune होने की स्थिति में पंक्तियों की गिनती करें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT COUNT(*) FROM repairs WHERE department = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT COUNT(*) FROM repairs WHERE department = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a888840d69a23410b604_hi", "language": "hi", "backend": "supabase", "operation": "top", "question": "repairs में quantity के उच्चतम मानों वाली 5 पंक्तियाँ प्रदर्शित करें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs ORDER BY quantity DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs ORDER BY quantity DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4996e2672571480e6c60_hi", "language": "hi", "backend": "mysql", "operation": "project", "question": "मुझे rentals का customer_name चाहिए, बाकी कुछ नहीं।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT customer_name FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT customer_name FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4217b243f42c7af06169_hi", "language": "hi", "backend": "mysql", "operation": "lt", "question": "enrollments में वे पंक्तियाँ खोजें जहाँ cost 10 से कम है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM enrollments WHERE cost < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments WHERE cost < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b654c9f6e4c53883151c_en", "language": "en", "backend": "supabase", "operation": "join", "question": "Present repairs.description and repairs_groups.label by associating repairs.group_id with repairs_groups.id.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT repairs.description, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT repairs.description, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9ef4d4f5d5bcf5858087_hi", "language": "hi", "backend": "supabase", "operation": "and", "question": "repairs से वे पंक्तियाँ प्रदर्शित करें जहाँ city Electronics है और price 10 से अधिक है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE city = 'Electronics' AND price > 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE city = 'Electronics' AND price > 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "03695bef6508dec8ebdb_hi", "language": "hi", "backend": "supabase", "operation": "join_filter", "question": "उन्हीं रिकॉर्ड्स का enrollments.customer_name लें जहाँ enrollments.group_id enrollments_groups.id से जुड़ा है और enrollments_groups.label Delhi के बराबर है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT enrollments.customer_name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT enrollments.customer_name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c2f17e8891f4e861494c_hinglish", "language": "hinglish", "backend": "mysql", "operation": "missing_value", "question": "Please repairs mein records ko department ke hisaab se filter karo, lekin maine koi value specify nahi ki hai.", "expected": {"action": "clarify", "question": "department ki kaunsi value chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"department ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "8cb6fe8c4ec5ccf02866_hinglish", "language": "hinglish", "backend": "mysql", "operation": "missing_value", "question": "Please deliveries mein records ko city ke hisaab se filter karo, lekin maine koi value specify nahi ki hai.", "expected": {"action": "clarify", "question": "city ki kaunsi value chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"city ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "71a64ddc45ae6b8498d9_hinglish", "language": "hinglish", "backend": "mysql", "operation": "final", "question": "Database tool ne 13 rows laayi hain. Kripya sankhya batao.", "expected": {"action": "answer", "text": "Tool ne 13 rows return ki."}, "output": "{\"action\":\"answer\",\"text\":\"Tool ne 13 rows return ki.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "21c860f0b85143a5adf6_hi", "language": "hi", "backend": "mysql", "operation": "max", "question": "deliveries में rating की अधिकतम संख्या क्या है?", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT MAX(rating) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT MAX(rating) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2c765382d84fd55136f4_hi", "language": "hi", "backend": "mysql", "operation": "group_count", "question": "rentals में प्रत्येक kind के पंक्ति काउंट दें।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM rentals GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM rentals GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c147e867055d7fd9f259_hi", "language": "hi", "backend": "mysql", "operation": "list_tables", "question": "कृपया इस डेटाबेस की तालिकाएँ fetch करें।", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "62e8127db1df87f17dfc_hi", "language": "hi", "backend": "mysql", "operation": "write", "question": "deliveries से हर एक पंक्ति को निकालें।", "expected": {"action": "clarify", "question": "यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "af4683485d1360b8afb9_hi", "language": "hi", "backend": "mysql", "operation": "max", "question": "repairs में quantity की अधिकतम संख्या क्या है?", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT MAX(quantity) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT MAX(quantity) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6391074b92a18a2d8964_hi", "language": "hi", "backend": "mysql", "operation": "project", "question": "मुझे enrollments का display_name चाहिए, बाकी कुछ नहीं।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT display_name FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT display_name FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b654c9f6e4c53883151c_hinglish", "language": "hinglish", "backend": "supabase", "operation": "join", "question": "repairs.group_id ko repairs_groups.id se join karke repairs.description aur repairs_groups.label ka data dikhao.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT repairs.description, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT repairs.description, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "cd68dcf59fb97c84247d_en", "language": "en", "backend": "mysql", "operation": "contains", "question": "Bring up the records in repairs where item_name features Mumbai, without regard to case.", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM repairs WHERE LOWER(item_name) LIKE LOWER('%Mumbai%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE LOWER(item_name) LIKE LOWER('%Mumbai%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2d039adc19e8ba87f706_hi", "language": "hi", "backend": "mysql", "operation": "or", "question": "deliveries से category Pune या North वाली पंक्तियाँ लें।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM deliveries WHERE category IN ('Pune', 'North');"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE category IN ('Pune', 'North');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3d12ba589211e391881d_hi", "language": "hi", "backend": "supabase", "operation": "not_null", "question": "repairs को फ़िल्टर करके सिर्फ वे पंक्तियाँ दिखाएं जहाँ title null नहीं है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE title IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE title IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b8b9810be462344512fe_hi", "language": "hi", "backend": "mysql", "operation": "min", "question": "deliveries में score का न्यूनतम स्तर क्या है?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT MIN(score) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(score) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a07055860746011ca946_hi", "language": "hi", "backend": "mysql", "operation": "project", "question": "मुझे enrollments का display_name चाहिए, बाकी कुछ नहीं।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT display_name FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5b11e37f42eb06caba31_hi", "language": "hi", "backend": "supabase", "operation": "not_null", "question": "deliveries को फ़िल्टर करके सिर्फ वे पंक्तियाँ दिखाएं जहाँ description null नहीं है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6bf08cbbd6b5cf97f755_hi", "language": "hi", "backend": "supabase", "operation": "sort_asc", "question": "repairs की सभी पंक्तियाँ cost के कम से ज्यादा क्रम में क्रमबद्ध करें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs ORDER BY cost ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs ORDER BY cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "467e53c90a42cd9935a3_hi", "language": "hi", "backend": "mysql", "operation": "lte", "question": "rentals का वो डेटा दिखाएं जहां price 5 से अधिक नहीं है।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM rentals WHERE price <= 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE price <= 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f34bb7005340164f75e6_en", "language": "en", "backend": "mysql", "operation": "final", "question": "The database tool gave 31 rows. What is the count?", "expected": {"action": "answer", "text": "The tool returned 31 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 31 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "f34bb7005340164f75e6_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "final", "question": "the database tool gave 31 rows. how many rows is that?", "expected": {"action": "answer", "text": "The tool returned 31 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 31 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "9955ba81eb264ff07981_hi", "language": "hi", "backend": "mysql", "operation": "ambiguous", "question": "deliveries के सर्वोत्तम रिकॉर्ड उजागर करें।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "f661788465b66e8a5822_hi", "language": "hi", "backend": "mysql", "operation": "lt", "question": "enrollments में वे पंक्तियाँ खोजें जहाँ score 200 से कम है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE score < 200;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE score < 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dcc973b271bcbc227307_hi", "language": "hi", "backend": "supabase", "operation": "and", "question": "repairs से वे पंक्तियाँ प्रदर्शित करें जहाँ region North है और balance 5 से अधिक है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE region = 'North' AND balance > 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE region = 'North' AND balance > 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8aeca0843f974e95bd97_hi", "language": "hi", "backend": "mysql", "operation": "list_tables", "question": "कृपया इस डेटाबेस की तालिकाएँ fetch करें।", "expected": {"action": "call", "name": "store.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"store.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "32989df4383378821b6c_hi", "language": "hi", "backend": "mysql", "operation": "describe", "question": "deliveries में मौजूद फील्ड्स और टाइप्स बताएं।", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "deliveries"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"deliveries\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "7942659b264a287de6f0_hinglish", "language": "hinglish", "backend": "mysql", "operation": "missing_value", "question": "Please repairs mein records ko department ke hisaab se filter karo, lekin maine koi value specify nahi ki hai.", "expected": {"action": "clarify", "question": "department ki kaunsi value chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"department ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "d00f22da639cc816ec58_hi", "language": "hi", "backend": "supabase", "operation": "final", "question": "डेटाबेस टूल द्वारा प्रदान की गई पंक्तियों की संख्या 114 है। मुझे यह संख्या बताएं।", "expected": {"action": "answer", "text": "टूल ने 114 पंक्तियाँ लौटाईं।"}, "output": "{\"action\":\"answer\",\"text\":\"टूल ने 114 पंक्तियाँ लौटाईं।\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "328b77dd66cdf8cabfd8_hi", "language": "hi", "backend": "mysql", "operation": "or", "question": "repairs से status O'Reilly या South वाली पंक्तियाँ लें।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs WHERE status IN ('O''Reilly', 'South');"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE status IN ('O''Reilly', 'South');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f34bb7005340164f75e6_hinglish", "language": "hinglish", "backend": "mysql", "operation": "final", "question": "Database tool ne 31 rows laayi hain. Kripya sankhya batao.", "expected": {"action": "answer", "text": "Tool ne 31 rows return ki."}, "output": "{\"action\":\"answer\",\"text\":\"Tool ne 31 rows return ki.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "90eda6200c21cebc9077_hi", "language": "hi", "backend": "mysql", "operation": "group_count", "question": "deliveries में प्रत्येक status के पंक्ति काउंट दें।", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT status, COUNT(*) FROM deliveries GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT status, COUNT(*) FROM deliveries GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f2eb08449a82d31f500e_hi", "language": "hi", "backend": "supabase", "operation": "missing_value", "question": "कृपया repairs में रिकॉर्ड्स को category के अनुसार फ़िल्टर करें, लेकिन मैंने कोई मान निर्दिष्ट नहीं किया है।", "expected": {"action": "clarify", "question": "category का कौन सा मान चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"category का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "fb320ffbb834f3c0a759_hi", "language": "hi", "backend": "mysql", "operation": "gte", "question": "rentals की वे एंट्री प्राप्त करें जहाँ cost 1 या उससे ऊपर है।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM rentals WHERE cost >= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE cost >= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f0a44c49f89e568f5a97_hi", "language": "hi", "backend": "supabase", "operation": "join_filter", "question": "उन्हीं रिकॉर्ड्स का repairs.name लें जहाँ repairs.group_id repairs_groups.id से जुड़ा है और repairs_groups.label Delhi के बराबर है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT repairs.name FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id WHERE repairs_groups.label = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT repairs.name FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id WHERE repairs_groups.label = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2a70571066edacbb19ac_hi", "language": "hi", "backend": "supabase", "operation": "bottom", "question": "rentals से score के आधार पर 5 न्यूनतम पंक्तियां, सबसे कम मान से शुरू होकर, प्रदर्शित करें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_rentals", "query": "SELECT * FROM rentals ORDER BY score ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_rentals\",\"query\":\"SELECT * FROM rentals ORDER BY score ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3b0a2346d3ae970242c1_hi", "language": "hi", "backend": "mysql", "operation": "year", "question": "2023 के recorded_at के आधार पर deliveries से डेटा निकालें।", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM deliveries WHERE YEAR(recorded_at) = 2023;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries WHERE YEAR(recorded_at) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f20e916f7b473213aa0d_hi", "language": "hi", "backend": "supabase", "operation": "and", "question": "deliveries से वे पंक्तियाँ प्रदर्शित करें जहाँ kind O'Reilly है और rating 500 से अधिक है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE kind = 'O''Reilly' AND rating > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE kind = 'O''Reilly' AND rating > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bc6f054aa3be632d1bce_hi", "language": "hi", "backend": "mysql", "operation": "distinct", "question": "enrollments में मौजूद डेटा से status के विशिष्ट मान निकालें।", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT DISTINCT status FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT DISTINCT status FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "848c830d91fb516d191b_hi", "language": "hi", "backend": "supabase", "operation": "not_null", "question": "repairs को फ़िल्टर करके सिर्फ वे पंक्तियाँ दिखाएं जहाँ item_name null नहीं है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs WHERE item_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs WHERE item_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "96006b107903118566cc_hi", "language": "hi", "backend": "mysql", "operation": "group_count", "question": "deliveries में प्रत्येक kind के पंक्ति काउंट दें।", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT kind, COUNT(*) FROM deliveries GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT kind, COUNT(*) FROM deliveries GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0e7660955234aff5d69a_hi", "language": "hi", "backend": "mysql", "operation": "group_sum", "question": "मुझे rentals में प्रत्येक category के लिए salary का कुल दिखाना है।", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT category, SUM(salary) FROM rentals GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT category, SUM(salary) FROM rentals GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d8445da89ec6e3d5a191_hi", "language": "hi", "backend": "mysql", "operation": "distinct", "question": "rentals में मौजूद डेटा से category के विशिष्ट मान निकालें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT DISTINCT category FROM rentals;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT category FROM rentals;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b0695b0692af5f9ff89e_hi", "language": "hi", "backend": "mysql", "operation": "gte", "question": "enrollments की वे एंट्री प्राप्त करें जहाँ score 10000 या उससे ऊपर है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE score >= 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE score >= 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "316b23c5d7f686804e72_hi", "language": "hi", "backend": "mysql", "operation": "sum", "question": "repairs एंटीटी के लिए score का कुल आउटपुट दें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT SUM(score) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT SUM(score) FROM repairs;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2fae606ab30f319280e2_hi", "language": "hi", "backend": "mysql", "operation": "month", "question": "rentals में order_date 12 महीने में होने वाली पंक्तियां दिखाएं।", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM rentals WHERE MONTH(order_date) = 12;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE MONTH(order_date) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9b0921e93df3d40ae856_hi", "language": "hi", "backend": "supabase", "operation": "bottom", "question": "enrollments से balance के आधार पर 1 न्यूनतम पंक्तियां, सबसे कम मान से शुरू होकर, प्रदर्शित करें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments ORDER BY balance ASC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments ORDER BY balance ASC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3920dafebc98270ffed5_hi", "language": "hi", "backend": "mysql", "operation": "between", "question": "enrollments से वे पंक्तियाँ प्रदर्शित करें जहाँ score >= 500 और <= 1000 है।", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM enrollments WHERE score BETWEEN 500 AND 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE score BETWEEN 500 AND 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "17da76d72f74206f6d3b_hi", "language": "hi", "backend": "supabase", "operation": "bottom", "question": "enrollments से salary के आधार पर 20 न्यूनतम पंक्तियां, सबसे कम मान से शुरू होकर, प्रदर्शित करें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_enrollments", "query": "SELECT * FROM enrollments ORDER BY salary ASC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_enrollments\",\"query\":\"SELECT * FROM enrollments ORDER BY salary ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6d9c843a92d3ec7fb99c_hi", "language": "hi", "backend": "mysql", "operation": "sort_desc", "question": "rentals से सभी पंक्तियाँ score के उच्चतम से निम्नतम क्रम में लौटाएँ।", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM rentals ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM rentals ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2e9ccfbc4c276796c15f_hi", "language": "hi", "backend": "mysql", "operation": "lte", "question": "deliveries का वो डेटा दिखाएं जहां balance 200 से अधिक नहीं है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE balance <= 200;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE balance <= 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c7aa166bcab44f298306_hi", "language": "hi", "backend": "mysql", "operation": "top", "question": "enrollments में cost के उच्चतम मानों वाली 1 पंक्तियाँ प्रदर्शित करें।", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM enrollments ORDER BY cost DESC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments ORDER BY cost DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b6da31e0b488a9ecb9e4_hi", "language": "hi", "backend": "mysql", "operation": "count_eq", "question": "rentals में region Books होने की स्थिति में पंक्तियों की गिनती करें।", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT COUNT(*) FROM rentals WHERE region = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM rentals WHERE region = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "660dd90727fa904c6f30_hi", "language": "hi", "backend": "mysql", "operation": "top", "question": "rentals में cost के उच्चतम मानों वाली 10 पंक्तियाँ प्रदर्शित करें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM rentals ORDER BY cost DESC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM rentals ORDER BY cost DESC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "78e1c795f40293f04ae5_hi", "language": "hi", "backend": "mysql", "operation": "sum", "question": "deliveries एंटीटी के लिए quantity का कुल आउटपुट दें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT SUM(quantity) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT SUM(quantity) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e8d5bf3c1d106a2fb229_hi", "language": "hi", "backend": "mysql", "operation": "sort_desc", "question": "rentals से सभी पंक्तियाँ rating के उच्चतम से निम्नतम क्रम में लौटाएँ।", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM rentals ORDER BY rating DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM rentals ORDER BY rating DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6a9265d659c9399f1229_hi", "language": "hi", "backend": "mysql", "operation": "group_sum", "question": "मुझे enrollments में प्रत्येक status के लिए salary का कुल दिखाना है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT status, SUM(salary) FROM enrollments GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT status, SUM(salary) FROM enrollments GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0e015b584f6a6179890c_hi", "language": "hi", "backend": "mysql", "operation": "date_after", "question": "rentals से वे रिकॉर्ड दिखाएं जहाँ created_at 2023-05-15 से पहले नहीं है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM rentals WHERE created_at >= '2023-05-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE created_at >= '2023-05-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ed547255667d7a6cfb80_en", "language": "en", "backend": "supabase", "operation": "join_filter", "question": "Select repairs.customer_name from repairs joined with repairs_groups on repairs.group_id where repairs_groups.label is South.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT repairs.customer_name FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id WHERE repairs_groups.label = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT repairs.customer_name FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id WHERE repairs_groups.label = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ed547255667d7a6cfb80_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "join_filter", "question": "select repairs.customer_name from repairs joined to repairs_groups on repairs.group_id where repairs_groups.label is South please", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT repairs.customer_name FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id WHERE repairs_groups.label = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT repairs.customer_name FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id WHERE repairs_groups.label = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "50120f3b3fbdc0e48064_hi", "language": "hi", "backend": "supabase", "operation": "final", "question": "डेटाबेस टूल द्वारा प्रदान की गई पंक्तियों की संख्या 143 है। मुझे यह संख्या बताएं।", "expected": {"action": "answer", "text": "टूल ने 143 पंक्तियाँ लौटाईं।"}, "output": "{\"action\":\"answer\",\"text\":\"टूल ने 143 पंक्तियाँ लौटाईं।\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "eab3dede73f7980dd2da_hi", "language": "hi", "backend": "mysql", "operation": "eq", "question": "enrollments से पूछें और वे सभी पंक्तियाँ दिखाएँ जहाँ department Mumbai है।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM enrollments WHERE department = 'Mumbai';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE department = 'Mumbai';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f637ccd41642ab190d04_hi", "language": "hi", "backend": "mysql", "operation": "null", "question": "repairs में customer_name null होने वाली पंक्तियाँ लाएँ।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs WHERE customer_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE customer_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5cf994b39a1eb0a5d0b6_hi", "language": "hi", "backend": "mysql", "operation": "null", "question": "deliveries में item_name null होने वाली पंक्तियाँ लाएँ।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM deliveries WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dc2e72d9a61704c6679c_hi", "language": "hi", "backend": "mysql", "operation": "top", "question": "enrollments में quantity के उच्चतम मानों वाली 10 पंक्तियाँ प्रदर्शित करें।", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM enrollments ORDER BY quantity DESC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments ORDER BY quantity DESC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c3d3a3d5475ef70e2f2c_hi", "language": "hi", "backend": "mysql", "operation": "sort_asc", "question": "enrollments की सभी पंक्तियाँ price के कम से ज्यादा क्रम में क्रमबद्ध करें।", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM enrollments ORDER BY price ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments ORDER BY price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9992776e0fbabf4da80e_hi", "language": "hi", "backend": "mysql", "operation": "month", "question": "rentals में created_at 12 महीने में होने वाली पंक्तियां दिखाएं।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM rentals WHERE MONTH(created_at) = 12;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE MONTH(created_at) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e213d3002d5c7836cc0a_hi", "language": "hi", "backend": "mysql", "operation": "between", "question": "repairs से वे पंक्तियाँ प्रदर्शित करें जहाँ amount >= 1000 और <= 1100 है।", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM repairs WHERE amount BETWEEN 1000 AND 1100;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE amount BETWEEN 1000 AND 1100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e309fa17d1bc158b8c8f_hi", "language": "hi", "backend": "mysql", "operation": "lte", "question": "deliveries का वो डेटा दिखाएं जहां quantity 5 से अधिक नहीं है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries WHERE quantity <= 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE quantity <= 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7b6982f2662ad49d01d8_hi", "language": "hi", "backend": "mysql", "operation": "count", "question": "enrollments में स्टोर की गई पंक्तियों की संख्या क्या है?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM enrollments;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM enrollments;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a89baf1622de706da189_hi", "language": "hi", "backend": "mysql", "operation": "and", "question": "rentals से वे पंक्तियाँ प्रदर्शित करें जहाँ status Pune है और amount 100 से अधिक है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM rentals WHERE status = 'Pune' AND amount > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM rentals WHERE status = 'Pune' AND amount > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "2daba35926021b3697eb_hi", "language": "hi", "backend": "mysql", "operation": "date_after", "question": "enrollments से वे रिकॉर्ड दिखाएं जहाँ recorded_at 2023-05-15 से पहले नहीं है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE recorded_at >= '2023-05-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE recorded_at >= '2023-05-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9f4ae4d3b48c49a8c943_hi", "language": "hi", "backend": "mysql", "operation": "gte", "question": "deliveries की वे एंट्री प्राप्त करें जहाँ salary 2 या उससे ऊपर है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM deliveries WHERE salary >= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries WHERE salary >= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ed547255667d7a6cfb80_hinglish", "language": "hinglish", "backend": "supabase", "operation": "join_filter", "question": "Un rows ka repairs.customer_name lao jahan repairs.group_id repairs_groups.id se juda hai aur repairs_groups.label South ke barabar hai.", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT repairs.customer_name FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id WHERE repairs_groups.label = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT repairs.customer_name FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id WHERE repairs_groups.label = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "00e1fcbe466a7fdf0d8a_hi", "language": "hi", "backend": "mysql", "operation": "date_after", "question": "rentals से वे रिकॉर्ड दिखाएं जहाँ order_date 2023-11-15 से पहले नहीं है।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM rentals WHERE order_date >= '2023-11-15';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE order_date >= '2023-11-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e0b7adca29f0eb32fe2c_hi", "language": "hi", "backend": "mysql", "operation": "between", "question": "deliveries से वे पंक्तियाँ प्रदर्शित करें जहाँ quantity >= 3 और <= 53 है।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM deliveries WHERE quantity BETWEEN 3 AND 53;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE quantity BETWEEN 3 AND 53;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7dfb47d110d38ff48e06_hi", "language": "hi", "backend": "mysql", "operation": "not_null", "question": "enrollments को फ़िल्टर करके सिर्फ वे पंक्तियाँ दिखाएं जहाँ title null नहीं है।", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM enrollments WHERE title IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments WHERE title IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "640719c646d2ad1c92e6_hi", "language": "hi", "backend": "mysql", "operation": "sort_asc", "question": "rentals की सभी पंक्तियाँ rating के कम से ज्यादा क्रम में क्रमबद्ध करें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM rentals ORDER BY rating ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM rentals ORDER BY rating ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1c48c9c4894db5982e9d_hi", "language": "hi", "backend": "mysql", "operation": "sql_error", "question": "क्योंकि अंतिम क्वेरी अज्ञात कॉलम के कारण फेल हुई, rentals स्कीमा का निरीक्षण करने के बाद ही दोबारा प्रयास करें।", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "rentals"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"rentas\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
{"id": "01072ae077b62a8a9a43_hi", "language": "hi", "backend": "mysql", "operation": "sort_desc", "question": "enrollments से सभी पंक्तियाँ salary के उच्चतम से निम्नतम क्रम में लौटाएँ।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM enrollments ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3f2bcb817dc0ac7fb510_hi", "language": "hi", "backend": "mysql", "operation": "sort_asc", "question": "enrollments की सभी पंक्तियाँ score के कम से ज्यादा क्रम में क्रमबद्ध करें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments ORDER BY score ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments ORDER BY score ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c752dd15f0913fe63436_hi", "language": "hi", "backend": "supabase", "operation": "bottom", "question": "repairs से cost के आधार पर 5 न्यूनतम पंक्तियां, सबसे कम मान से शुरू होकर, प्रदर्शित करें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT * FROM repairs ORDER BY cost ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT * FROM repairs ORDER BY cost ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "3575acb51007208adc3f_hi", "language": "hi", "backend": "mysql", "operation": "not_null", "question": "rentals को फ़िल्टर करके सिर्फ वे पंक्तियाँ दिखाएं जहाँ description null नहीं है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM rentals WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM rentals WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "619f53e951d6c4a5acdb_hi", "language": "hi", "backend": "supabase", "operation": "bottom", "question": "deliveries से quantity के आधार पर 1 न्यूनतम पंक्तियां, सबसे कम मान से शुरू होकर, प्रदर्शित करें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries ORDER BY quantity ASC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries ORDER BY quantity ASC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "9b4ac5b2bb0340639986_hi", "language": "hi", "backend": "mysql", "operation": "month", "question": "enrollments में order_date 6 महीने में होने वाली पंक्तियां दिखाएं।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE MONTH(order_date) = 6;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE MONTH(order_date) = 6;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "59fa7ad459e87198d29d_hi", "language": "hi", "backend": "mysql", "operation": "not_null", "question": "enrollments को फ़िल्टर करके सिर्फ वे पंक्तियाँ दिखाएं जहाँ title null नहीं है।", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM enrollments WHERE title IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE title IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "13ac9d307d6b101b8705_hi", "language": "hi", "backend": "mysql", "operation": "gte", "question": "repairs की वे एंट्री प्राप्त करें जहाँ cost 0 या उससे ऊपर है।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs WHERE cost >= 0;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE cost >= 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8926f167d28d5c555478_hi", "language": "hi", "backend": "mysql", "operation": "gte", "question": "deliveries की वे एंट्री प्राप्त करें जहाँ quantity 50 या उससे ऊपर है।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM deliveries WHERE quantity >= 50;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE quantity >= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6569be8d680d87bb283d_hi", "language": "hi", "backend": "mysql", "operation": "gt", "question": "मुझे deliveries से वे पंक्तियाँ चाहिए जहाँ amount 200 से अधिक है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM deliveries WHERE amount > 200;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries WHERE amount > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dd9565e510782b49a5f0_hi", "language": "hi", "backend": "mysql", "operation": "count", "question": "deliveries में स्टोर की गई पंक्तियों की संख्या क्या है?", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT COUNT(*) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "8d03186d0c3faa663273_hi", "language": "hi", "backend": "mysql", "operation": "month", "question": "deliveries में order_date 1 महीने में होने वाली पंक्तियां दिखाएं।", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM deliveries WHERE MONTH(order_date) = 1;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE MONTH(order_date) = 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1339ee90b3d8b8eac1f7_en", "language": "en", "backend": "mysql", "operation": "join_filter", "question": "Select enrollments.name from enrollments joined with enrollments_groups on enrollments.group_id where enrollments_groups.label is North.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT enrollments.name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT enrollments.name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1339ee90b3d8b8eac1f7_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "join_filter", "question": "select enrollments.name from enrollments joined to enrollments_groups on enrollments.group_id where enrollments_groups.label is North please", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT enrollments.name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT enrollments.name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "e053da65d9fd664996e6_hi", "language": "hi", "backend": "mysql", "operation": "gt", "question": "मुझे repairs से वे पंक्तियाँ चाहिए जहाँ amount 20 से अधिक है।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM repairs WHERE amount > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM repairs WHERE amount > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fc1a5aaf55bd30034ad0_hi", "language": "hi", "backend": "mysql", "operation": "count", "question": "deliveries में स्टोर की गई पंक्तियों की संख्या क्या है?", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM deliveries;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "029cd7c058965ed14b05_hi", "language": "hi", "backend": "mysql", "operation": "eq", "question": "repairs से पूछें और वे सभी पंक्तियाँ दिखाएँ जहाँ department active है।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs WHERE department = 'active';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE department = 'active';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "384c3b042d68b5a13b59_hi", "language": "hi", "backend": "mysql", "operation": "between", "question": "deliveries से वे पंक्तियाँ प्रदर्शित करें जहाँ salary >= 200 और <= 300 है।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM deliveries WHERE salary BETWEEN 200 AND 300;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE salary BETWEEN 200 AND 300;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "202d7bd6d9a637a071e8_hi", "language": "hi", "backend": "mysql", "operation": "between", "question": "deliveries से वे पंक्तियाँ प्रदर्शित करें जहाँ cost >= 50 और <= 60 है।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM deliveries WHERE cost BETWEEN 50 AND 60;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM deliveries WHERE cost BETWEEN 50 AND 60;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "68d0c440e1210d5f5590_hi", "language": "hi", "backend": "mysql", "operation": "group_sum", "question": "मुझे repairs में प्रत्येक kind के लिए score का कुल दिखाना है।", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT kind, SUM(score) FROM repairs GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT kind, SUM(score) FROM repairs GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "d9f1ee5cf1a15f1ad44b_hi", "language": "hi", "backend": "mysql", "operation": "eq", "question": "repairs से पूछें और वे सभी पंक्तियाँ दिखाएँ जहाँ department Books है।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT * FROM repairs WHERE department = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM repairs WHERE department = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "04165cd31bcf23c6fac9_hi", "language": "hi", "backend": "mysql", "operation": "eq", "question": "repairs से पूछें और वे सभी पंक्तियाँ दिखाएँ जहाँ category O'Reilly है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM repairs WHERE category = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE category = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "33d0ab3caa0fba0e3985_hi", "language": "hi", "backend": "mysql", "operation": "count", "question": "repairs में स्टोर की गई पंक्तियों की संख्या क्या है?", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT COUNT(*) FROM repairs;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM repairs WHERE kind = 'Agra';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
{"id": "dae8c9c830c197ee8e74_hi", "language": "hi", "backend": "mysql", "operation": "sort_desc", "question": "repairs से सभी पंक्तियाँ salary के उच्चतम से निम्नतम क्रम में लौटाएँ।", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM repairs ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM repairs ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b654c9f6e4c53883151c_hi", "language": "hi", "backend": "supabase", "operation": "join", "question": "repairs.group_id को repairs_groups.id से जोड़कर repairs.description और repairs_groups.label का डेटा दिखाएँ।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT repairs.description, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT repairs.description, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "bff6b3ccadd4c292165c_hi", "language": "hi", "backend": "mysql", "operation": "sql_error", "question": "क्योंकि अंतिम क्वेरी अज्ञात कॉलम के कारण फेल हुई, deliveries स्कीमा का निरीक्षण करने के बाद ही दोबारा प्रयास करें।", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "deliveries"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"deliveries\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "0f9f5d9f2fda4132f1bf_hi", "language": "hi", "backend": "supabase", "operation": "contains", "question": "deliveries में display_name में Mumbai रखने वाली पंक्तियाँ सूचीबद्ध करें, मामला-असंवेदनशीलता के साथ।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_deliveries", "query": "SELECT * FROM deliveries WHERE LOWER(display_name) LIKE LOWER('%Mumbai%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_deliveries\",\"query\":\"SELECT * FROM deliveries WHERE LOWER(display_name) LIKE LOWER('%Mumbai%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f12c053fdb90a634ce20_hi", "language": "hi", "backend": "mysql", "operation": "sql_error", "question": "क्योंकि अंतिम क्वेरी अज्ञात कॉलम के कारण फेल हुई, repairs स्कीमा का निरीक्षण करने के बाद ही दोबारा प्रयास करें।", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "repairs"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"repairs\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "9325b9d4f4ee1c176f2f_hi", "language": "hi", "backend": "mysql", "operation": "gt", "question": "मुझे repairs से वे पंक्तियाँ चाहिए जहाँ score 1500 से अधिक है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT * FROM repairs WHERE score > 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM repairs WHERE score > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "dafbcd52face40a7b678_hi", "language": "hi", "backend": "mysql", "operation": "group_sum", "question": "मुझे repairs में प्रत्येक kind के लिए rating का कुल दिखाना है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT kind, SUM(rating) FROM repairs GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT kind, SUM(rating) FROM repairs GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0aec7748ac00fd2195f1_en", "language": "en", "backend": "mysql", "operation": "join_filter", "question": "Select rentals.item_name from rentals joined with rentals_groups on rentals.group_id where rentals_groups.label is North.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT rentals.item_name FROM rentals JOIN rentals_groups ON rentals.group_id = rentals_groups.id WHERE rentals_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT rentals.item_name FROM rentals JOIN rentals_groups ON rentals.group_id = rentals_groups.id WHERE rentals_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0aec7748ac00fd2195f1_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "join_filter", "question": "select rentals.item_name from rentals joined to rentals_groups on rentals.group_id where rentals_groups.label is North please", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT rentals.item_name FROM rentals JOIN rentals_groups ON rentals.group_id = rentals_groups.id WHERE rentals_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT rentals.item_name FROM rentals JOIN rentals_groups ON rentals.group_id = rentals_groups.id WHERE rentals_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "cd2773edfc61c2b4bc9a_hi", "language": "hi", "backend": "mysql", "operation": "group_sum", "question": "मुझे deliveries में प्रत्येक department के लिए rating का कुल दिखाना है।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT department, SUM(rating) FROM deliveries GROUP BY department;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT department, SUM(rating) FROM deliveries GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1339ee90b3d8b8eac1f7_hinglish", "language": "hinglish", "backend": "mysql", "operation": "join_filter", "question": "Un rows ka enrollments.name lao jahan enrollments.group_id enrollments_groups.id se juda hai aur enrollments_groups.label North ke barabar hai.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT enrollments.name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT enrollments.name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "4a26377ced6478bb8d0e_hi", "language": "hi", "backend": "mysql", "operation": "and", "question": "repairs से वे पंक्तियाँ प्रदर्शित करें जहाँ city Electronics है और quantity 200 से अधिक है।", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM repairs WHERE city = 'Electronics' AND quantity > 200;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM repairs WHERE city = 'Electronics' AND quantity > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7839648878bf9feda17b_hi", "language": "hi", "backend": "mysql", "operation": "count_eq", "question": "repairs में department Pune होने की स्थिति में पंक्तियों की गिनती करें।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT COUNT(*) FROM repairs WHERE department = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM repairs WHERE department = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "efee860c507a44ecafbb_hi", "language": "hi", "backend": "mysql", "operation": "sql_error", "question": "क्योंकि अंतिम क्वेरी अज्ञात कॉलम के कारण फेल हुई, deliveries स्कीमा का निरीक्षण करने के बाद ही दोबारा प्रयास करें।", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "deliveries"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"deliveries\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "dbc778be5f23038bcc3d_hi", "language": "hi", "backend": "mysql", "operation": "count_eq", "question": "repairs में category O'Reilly होने की स्थिति में पंक्तियों की गिनती करें।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT COUNT(*) FROM repairs WHERE category = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM repairs WHERE category = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "5e4c3931848b0640bfaa_hi", "language": "hi", "backend": "mysql", "operation": "top", "question": "deliveries में quantity के उच्चतम मानों वाली 10 पंक्तियाँ प्रदर्शित करें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries ORDER BY quantity DESC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries ORDER BY quantity DESC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "68ec6ad5b5ff4ebdc86a_hi", "language": "hi", "backend": "mysql", "operation": "sort_asc", "question": "deliveries की सभी पंक्तियाँ balance के कम से ज्यादा क्रम में क्रमबद्ध करें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM deliveries ORDER BY balance ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM deliveries ORDER BY balance ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "928e148fdcacd00b97f8_hi", "language": "hi", "backend": "mysql", "operation": "sql_error", "question": "क्योंकि अंतिम क्वेरी अज्ञात कॉलम के कारण फेल हुई, deliveries स्कीमा का निरीक्षण करने के बाद ही दोबारा प्रयास करें।", "expected": {"action": "call", "name": "store.describe_table", "arguments": {"table": "deliveries"}}, "output": "{\"action\":\"call\",\"name\":\"store.describe_table\",\"arguments\":{\"table\":\"deliveries\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "0e9353d09ce2f0cd9994_hi", "language": "hi", "backend": "mysql", "operation": "count_eq", "question": "deliveries में city O'Reilly होने की स्थिति में पंक्तियों की गिनती करें।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"sql": "SELECT COUNT(*) FROM deliveries WHERE city = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM deliveries WHERE city = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1667f375dfed0ad934a4_hi", "language": "hi", "backend": "mysql", "operation": "and", "question": "repairs से वे पंक्तियाँ प्रदर्शित करें जहाँ city Books है और rating 1500 से अधिक है।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT * FROM repairs WHERE city = 'Books' AND rating > 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE city = 'Books' AND rating > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7cf5c995d18de3fe3679_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "join", "question": "retrieve repairs.display_name and repairs_groups.label join repairs.group_id to repairs_groups.id", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT repairs.display_name, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT repairs.display_name, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ebc657ab9bc6f3f33d21_hi", "language": "hi", "backend": "supabase", "operation": "having", "question": "repairs से 1000 पंक्तियों से अधिक वाले department समूहों को उनकी पंक्ति संख्या के साथ दिखाइए।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT department, COUNT(*) FROM repairs GROUP BY department HAVING COUNT(*) > 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT department, COUNT(*) FROM repairs GROUP BY department HAVING COUNT(*) > 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c721428d10124e8c39fe_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "join", "question": "retrieve repairs.display_name and repairs_groups.label join repairs.group_id to repairs_groups.id", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT repairs.display_name, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT repairs.display_name, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a9bada7658db91098256_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "join", "question": "retrieve deliveries.customer_name and deliveries_groups.label join deliveries.group_id to deliveries_groups.id", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT deliveries.customer_name, deliveries_groups.label FROM deliveries JOIN deliveries_groups ON deliveries.group_id = deliveries_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT deliveries.customer_name, deliveries_groups.label FROM deliveries JOIN deliveries_groups ON deliveries.group_id = deliveries_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "0aec7748ac00fd2195f1_hinglish", "language": "hinglish", "backend": "mysql", "operation": "join_filter", "question": "Un rows ka rentals.item_name lao jahan rentals.group_id rentals_groups.id se juda hai aur rentals_groups.label North ke barabar hai.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT rentals.item_name FROM rentals JOIN rentals_groups ON rentals.group_id = rentals_groups.id WHERE rentals_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT rentals.item_name FROM rentals JOIN rentals_groups ON rentals.group_id = rentals_groups.id WHERE rentals_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7cf5c995d18de3fe3679_en", "language": "en", "backend": "mysql", "operation": "join", "question": "Present repairs.display_name and repairs_groups.label by associating repairs.group_id with repairs_groups.id.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT repairs.display_name, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT repairs.display_name, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c721428d10124e8c39fe_en", "language": "en", "backend": "mysql", "operation": "join", "question": "Present repairs.display_name and repairs_groups.label by associating repairs.group_id with repairs_groups.id.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT repairs.display_name, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT repairs.display_name, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a9bada7658db91098256_en", "language": "en", "backend": "mysql", "operation": "join", "question": "Present deliveries.customer_name and deliveries_groups.label by associating deliveries.group_id with deliveries_groups.id.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT deliveries.customer_name, deliveries_groups.label FROM deliveries JOIN deliveries_groups ON deliveries.group_id = deliveries_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT deliveries.customer_name, deliveries_groups.label FROM deliveries JOIN deliveries_groups ON deliveries.group_id = deliveries_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "eb4c2f1cc33712f0b82e_hi", "language": "hi", "backend": "mysql", "operation": "bottom", "question": "enrollments से cost के आधार पर 5 न्यूनतम पंक्तियां, सबसे कम मान से शुरू होकर, प्रदर्शित करें।", "expected": {"action": "call", "name": "store.query", "arguments": {"sql": "SELECT * FROM enrollments ORDER BY cost ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"sql\":\"SELECT * FROM enrollments ORDER BY cost ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7cf5c995d18de3fe3679_hinglish", "language": "hinglish", "backend": "mysql", "operation": "join", "question": "repairs.group_id ko repairs_groups.id se join karke repairs.display_name aur repairs_groups.label ka data dikhao.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT repairs.display_name, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT repairs.display_name, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c721428d10124e8c39fe_hinglish", "language": "hinglish", "backend": "mysql", "operation": "join", "question": "repairs.group_id ko repairs_groups.id se join karke repairs.display_name aur repairs_groups.label ka data dikhao.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT repairs.display_name, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT repairs.display_name, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a9bada7658db91098256_hinglish", "language": "hinglish", "backend": "mysql", "operation": "join", "question": "deliveries.group_id ko deliveries_groups.id se join karke deliveries.customer_name aur deliveries_groups.label ka data dikhao.", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT deliveries.customer_name, deliveries_groups.label FROM deliveries JOIN deliveries_groups ON deliveries.group_id = deliveries_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT deliveries.customer_name, deliveries_groups.label FROM deliveries JOIN deliveries_groups ON deliveries.group_id = deliveries_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "b93dfa01b85949af4005_hi", "language": "hi", "backend": "mysql", "operation": "contains", "question": "deliveries में description में Mumbai रखने वाली पंक्तियाँ सूचीबद्ध करें, मामला-असंवेदनशीलता के साथ।", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM deliveries WHERE LOWER(description) LIKE LOWER('%Mumbai%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM deliveries WHERE LOWER(description) LIKE LOWER('%Mumbai%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "57e4f37da2636ee67319_hi", "language": "hi", "backend": "mysql", "operation": "contains", "question": "enrollments में item_name में active रखने वाली पंक्तियाँ सूचीबद्ध करें, मामला-असंवेदनशीलता के साथ।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT * FROM enrollments WHERE LOWER(item_name) LIKE LOWER('%active%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM enrollments WHERE LOWER(item_name) LIKE LOWER('%active%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "64c0b80fc5c7f96a90b6_hi", "language": "hi", "backend": "supabase", "operation": "final", "question": "डेटाबेस टूल द्वारा प्रदान की गई पंक्तियों की संख्या 109 है। मुझे यह संख्या बताएं।", "expected": {"action": "answer", "text": "टूल ने 109 पंक्तियाँ लौटाईं।"}, "output": "{\"action\":\"answer\",\"text\":\"टूल ने 109 पंक्तियाँ लौटाईं।\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "b9d861b71317ad86ce3e_hi", "language": "hi", "backend": "mysql", "operation": "having", "question": "enrollments से 2 पंक्तियों से अधिक वाले category समूहों को उनकी पंक्ति संख्या के साथ दिखाइए।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM enrollments GROUP BY category HAVING COUNT(*) > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM enrollments GROUP BY category HAVING COUNT(*) > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "cd68dcf59fb97c84247d_hi", "language": "hi", "backend": "mysql", "operation": "contains", "question": "repairs में item_name में Mumbai रखने वाली पंक्तियाँ सूचीबद्ध करें, मामला-असंवेदनशीलता के साथ।", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT * FROM repairs WHERE LOWER(item_name) LIKE LOWER('%Mumbai%');"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT * FROM repairs WHERE LOWER(item_name) LIKE LOWER('%Mumbai%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "6aaf184e4cafcf2f5c16_hi", "language": "hi", "backend": "mysql", "operation": "having", "question": "repairs से 20 पंक्तियों से अधिक वाले category समूहों को उनकी पंक्ति संख्या के साथ दिखाइए।", "expected": {"action": "call", "name": "store.query", "arguments": {"query": "SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"store.query\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "502d4fde824d62a8f495_hi", "language": "hi", "backend": "supabase", "operation": "final", "question": "डेटाबेस टूल द्वारा प्रदान की गई पंक्तियों की संख्या 13 है। मुझे यह संख्या बताएं।", "expected": {"action": "answer", "text": "टूल ने 13 पंक्तियाँ लौटाईं।"}, "output": "{\"action\":\"answer\",\"text\":\"टूल ने 13 पंक्तियाँ लौटाईं।\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "c4a83c9634139691ddde_hi", "language": "hi", "backend": "mysql", "operation": "having", "question": "repairs से 200 पंक्तियों से अधिक वाले category समूहों को उनकी पंक्ति संख्या के साथ दिखाइए।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 200;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT category, COUNT(*) FROM repairs GROUP BY category HAVING COUNT(*) > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "ed547255667d7a6cfb80_hi", "language": "hi", "backend": "supabase", "operation": "join_filter", "question": "उन्हीं रिकॉर्ड्स का repairs.customer_name लें जहाँ repairs.group_id repairs_groups.id से जुड़ा है और repairs_groups.label South के बराबर है।", "expected": {"action": "call", "name": "store.execute_sql", "arguments": {"project_id": "demo_repairs", "query": "SELECT repairs.customer_name FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id WHERE repairs_groups.label = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"project_id\":\"demo_repairs\",\"query\":\"SELECT repairs.customer_name FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id WHERE repairs_groups.label = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "7cf5c995d18de3fe3679_hi", "language": "hi", "backend": "mysql", "operation": "join", "question": "repairs.group_id को repairs_groups.id से जोड़कर repairs.display_name और repairs_groups.label का डेटा दिखाएँ।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT repairs.display_name, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT repairs.display_name, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "c721428d10124e8c39fe_hi", "language": "hi", "backend": "mysql", "operation": "join", "question": "repairs.group_id को repairs_groups.id से जोड़कर repairs.display_name और repairs_groups.label का डेटा दिखाएँ।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT repairs.display_name, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT repairs.display_name, repairs_groups.label FROM repairs JOIN repairs_groups ON repairs.group_id = repairs_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "a9bada7658db91098256_hi", "language": "hi", "backend": "mysql", "operation": "join", "question": "deliveries.group_id को deliveries_groups.id से जोड़कर deliveries.customer_name और deliveries_groups.label का डेटा दिखाएँ।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT deliveries.customer_name, deliveries_groups.label FROM deliveries JOIN deliveries_groups ON deliveries.group_id = deliveries_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT deliveries.customer_name, deliveries_groups.label FROM deliveries JOIN deliveries_groups ON deliveries.group_id = deliveries_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "1339ee90b3d8b8eac1f7_hi", "language": "hi", "backend": "mysql", "operation": "join_filter", "question": "उन्हीं रिकॉर्ड्स का enrollments.name लें जहाँ enrollments.group_id enrollments_groups.id से जुड़ा है और enrollments_groups.label North के बराबर है।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"sql": "SELECT enrollments.name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"sql\":\"SELECT enrollments.name FROM enrollments JOIN enrollments_groups ON enrollments.group_id = enrollments_groups.id WHERE enrollments_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "fcd691cda0d5bb67b608_hi", "language": "hi", "backend": "mysql", "operation": "missing_value", "question": "कृपया rentals में रिकॉर्ड्स को city के अनुसार फ़िल्टर करें, लेकिन मैंने कोई मान निर्दिष्ट नहीं किया है।", "expected": {"action": "clarify", "question": "city का कौन सा मान चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"city का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "0aec7748ac00fd2195f1_hi", "language": "hi", "backend": "mysql", "operation": "join_filter", "question": "उन्हीं रिकॉर्ड्स का rentals.item_name लें जहाँ rentals.group_id rentals_groups.id से जुड़ा है और rentals_groups.label North के बराबर है।", "expected": {"action": "call", "name": "store.run_query", "arguments": {"query": "SELECT rentals.item_name FROM rentals JOIN rentals_groups ON rentals.group_id = rentals_groups.id WHERE rentals_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"store.run_query\",\"arguments\":{\"query\":\"SELECT rentals.item_name FROM rentals JOIN rentals_groups ON rentals.group_id = rentals_groups.id WHERE rentals_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
{"id": "f2d0b2d997d6372721b0_hi", "language": "hi", "backend": "mysql", "operation": "missing_value", "question": "कृपया enrollments में रिकॉर्ड्स को department के अनुसार फ़िल्टर करें, लेकिन मैंने कोई मान निर्दिष्ट नहीं किया है।", "expected": {"action": "clarify", "question": "department का कौन सा मान चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"department का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "c2f17e8891f4e861494c_hi", "language": "hi", "backend": "mysql", "operation": "missing_value", "question": "कृपया repairs में रिकॉर्ड्स को department के अनुसार फ़िल्टर करें, लेकिन मैंने कोई मान निर्दिष्ट नहीं किया है।", "expected": {"action": "clarify", "question": "department का कौन सा मान चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"department का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "8cb6fe8c4ec5ccf02866_hi", "language": "hi", "backend": "mysql", "operation": "missing_value", "question": "कृपया deliveries में रिकॉर्ड्स को city के अनुसार फ़िल्टर करें, लेकिन मैंने कोई मान निर्दिष्ट नहीं किया है।", "expected": {"action": "clarify", "question": "city का कौन सा मान चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"city का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "71a64ddc45ae6b8498d9_hi", "language": "hi", "backend": "mysql", "operation": "final", "question": "डेटाबेस टूल द्वारा प्रदान की गई पंक्तियों की संख्या 13 है। मुझे यह संख्या बताएं।", "expected": {"action": "answer", "text": "टूल ने 13 पंक्तियाँ लौटाईं।"}, "output": "{\"action\":\"answer\",\"text\":\"टूल ने 13 पंक्तियाँ लौटाईं।\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "7942659b264a287de6f0_hi", "language": "hi", "backend": "mysql", "operation": "missing_value", "question": "कृपया repairs में रिकॉर्ड्स को department के अनुसार फ़िल्टर करें, लेकिन मैंने कोई मान निर्दिष्ट नहीं किया है।", "expected": {"action": "clarify", "question": "department का कौन सा मान चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"department का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
{"id": "f34bb7005340164f75e6_hi", "language": "hi", "backend": "mysql", "operation": "final", "question": "डेटाबेस टूल द्वारा प्रदान की गई पंक्तियों की संख्या 31 है। मुझे यह संख्या बताएं।", "expected": {"action": "answer", "text": "टूल ने 31 पंक्तियाँ लौटाईं।"}, "output": "{\"action\":\"call\",\"name\":\"store.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM deliveries;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}}
{"id": "6575b299f08c87b34390_en", "language": "en", "backend": "supabase", "operation": "search", "question": "Look for indexes inside the documentation.", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "indexes"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6575b299f08c87b34390_hinglish", "language": "hinglish", "backend": "supabase", "operation": "search", "question": "Docs ke andar indexes dhoondo.", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "indexes"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6575b299f08c87b34390_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "search", "question": "help me find indexes in the documentation", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "indexes"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a150b9436cc95f4db00f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "weather", "question": "Delhi ka mausam fahrenheit mein lao.", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "e416c627b8dd4b3c2bfc_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ticket", "question": "display the record for TKT-9638", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-9638"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9638\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a150b9436cc95f4db00f_en", "language": "en", "backend": "supabase", "operation": "weather", "question": "Display the weather for Delhi using fahrenheit.", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "dd1576137b2d7927bedc_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ticket", "question": "display the record for TKT-286", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-286"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-286\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a150b9436cc95f4db00f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "weather", "question": "Please show me weather for Delhi in fahrenheit.", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "e416c627b8dd4b3c2bfc_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ticket", "question": "ticket TKT-9638 ki info retrieve karo.", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-9638"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9638\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "dd1576137b2d7927bedc_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ticket", "question": "ticket TKT-286 ki info retrieve karo.", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-286"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-286\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "19099c12f0e28cd0f874_hinglish", "language": "hinglish", "backend": "supabase", "operation": "read_file", "question": "/config/app.json par file load karo.", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"service.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a74cd18fb14dca57100e_hinglish", "language": "hinglish", "backend": "supabase", "operation": "read_file", "question": "/docs/README.md par file load karo.", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/docs/README.md"}}, "output": "{\"action\":\"call\",\"name\":\"service.read_file\",\"arguments\":{\"path\":\"/docs/README.md\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "e416c627b8dd4b3c2bfc_en", "language": "en", "backend": "supabase", "operation": "ticket", "question": "Please locate the information for support ticket TKT-9638.", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-9638"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9638\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "dd1576137b2d7927bedc_en", "language": "en", "backend": "supabase", "operation": "ticket", "question": "Please locate the information for support ticket TKT-286.", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-286"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-286\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a150b9436cc95f4db00f_hi", "language": "hi", "backend": "supabase", "operation": "weather", "question": "Delhi का मौसम fahrenheit में लाएं।", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "19099c12f0e28cd0f874_en", "language": "en", "backend": "supabase", "operation": "read_file", "question": "Load the content of the file at /config/app.json.", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"service.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "19099c12f0e28cd0f874_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "read_file", "question": "Load the file at /config/app.json and read it.", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"service.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a74cd18fb14dca57100e_en", "language": "en", "backend": "supabase", "operation": "read_file", "question": "Load the content of the file at /docs/README.md.", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/docs/README.md"}}, "output": "{\"action\":\"call\",\"name\":\"service.read_file\",\"arguments\":{\"path\":\"/docs/README.md\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a74cd18fb14dca57100e_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "read_file", "question": "Load the file at /docs/README.md and read it.", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/docs/README.md"}}, "output": "{\"action\":\"call\",\"name\":\"service.read_file\",\"arguments\":{\"path\":\"/docs/README.md\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "e416c627b8dd4b3c2bfc_hi", "language": "hi", "backend": "supabase", "operation": "ticket", "question": "टिकट TKT-9638 का डेटा फेच करें।", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-9638"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9638\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "dd1576137b2d7927bedc_hi", "language": "hi", "backend": "supabase", "operation": "ticket", "question": "टिकट TKT-286 का डेटा फेच करें।", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-286"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-286\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "6575b299f08c87b34390_hi", "language": "hi", "backend": "supabase", "operation": "search", "question": "मुझे दस्तावेज़ीकरण में indexes की आवश्यकता है।", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "indexes"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "19099c12f0e28cd0f874_hi", "language": "hi", "backend": "supabase", "operation": "read_file", "question": "/config/app.json पर स्थित फ़ाइल का सामग्री लोड करें।", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"service.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a74cd18fb14dca57100e_hi", "language": "hi", "backend": "supabase", "operation": "read_file", "question": "/docs/README.md पर स्थित फ़ाइल का सामग्री लोड करें।", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/docs/README.md"}}, "output": "{\"action\":\"call\",\"name\":\"service.read_file\",\"arguments\":{\"path\":\"/docs/README.md\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "32d9828b8b0cba285197_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ticket", "question": "display the record for TKT-7867", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-7867"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-7867\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "e91695694d804dc7cf13_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ticket", "question": "display the record for TKT-9915", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-9915"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9915\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "48a316f4c32067e3ca40_en", "language": "en", "backend": "supabase", "operation": "search", "question": "Look for row level security inside the documentation.", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "row level security"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "48a316f4c32067e3ca40_hinglish", "language": "hinglish", "backend": "supabase", "operation": "search", "question": "Docs ke andar row level security dhoondo.", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "row level security"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "32d9828b8b0cba285197_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ticket", "question": "ticket TKT-7867 ki info retrieve karo.", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-7867"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-7867\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "e91695694d804dc7cf13_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ticket", "question": "ticket TKT-9915 ki info retrieve karo.", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-9915"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9915\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "48a316f4c32067e3ca40_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "search", "question": "help me find row level security in the documentation", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "row level security"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "9c2fcd765bf5c2df33aa_en", "language": "en", "backend": "supabase", "operation": "search", "question": "Look for row level security inside the documentation.", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "row level security"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "9c2fcd765bf5c2df33aa_hinglish", "language": "hinglish", "backend": "supabase", "operation": "search", "question": "Docs ke andar row level security dhoondo.", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "row level security"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "32d9828b8b0cba285197_en", "language": "en", "backend": "supabase", "operation": "ticket", "question": "Please locate the information for support ticket TKT-7867.", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-7867"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-7867\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "e815e61cd1cbba2d7eec_hinglish", "language": "hinglish", "backend": "supabase", "operation": "read_file", "question": "/reports/sales.csv par file load karo.", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/reports/sales.csv"}}, "output": "{\"action\":\"call\",\"name\":\"service.read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "e91695694d804dc7cf13_en", "language": "en", "backend": "supabase", "operation": "ticket", "question": "Please locate the information for support ticket TKT-9915.", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-9915"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9915\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a9cd5b7c451419ebb9c3_hinglish", "language": "hinglish", "backend": "supabase", "operation": "read_file", "question": "/reports/sales.csv par file load karo.", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/reports/sales.csv"}}, "output": "{\"action\":\"call\",\"name\":\"service.service.read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false, "error": "Unknown tool"}}
{"id": "9c2fcd765bf5c2df33aa_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "search", "question": "help me find row level security in the documentation", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "row level security"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "070d5d78902fa837cb16_hinglish", "language": "hinglish", "backend": "supabase", "operation": "read_file", "question": "/notes/setup.txt par file load karo.", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"service.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "e815e61cd1cbba2d7eec_en", "language": "en", "backend": "supabase", "operation": "read_file", "question": "Load the content of the file at /reports/sales.csv.", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/reports/sales.csv"}}, "output": "{\"action\":\"call\",\"name\":\"service.read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "e815e61cd1cbba2d7eec_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "read_file", "question": "Load the file at /reports/sales.csv and read it.", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/reports/sales.csv"}}, "output": "{\"action\":\"call\",\"name\":\"service.read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a9cd5b7c451419ebb9c3_en", "language": "en", "backend": "supabase", "operation": "read_file", "question": "Load the content of the file at /reports/sales.csv.", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/reports/sales.csv"}}, "output": "{\"action\":\"call\",\"name\":\"service.read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a9cd5b7c451419ebb9c3_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "read_file", "question": "Load the file at /reports/sales.csv and read it.", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/reports/sales.csv"}}, "output": "{\"action\":\"call\",\"name\":\"service.read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "32d9828b8b0cba285197_hi", "language": "hi", "backend": "supabase", "operation": "ticket", "question": "टिकट TKT-7867 का डेटा फेच करें।", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-7867"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-7867\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "070d5d78902fa837cb16_en", "language": "en", "backend": "supabase", "operation": "read_file", "question": "Load the content of the file at /notes/setup.txt.", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"service.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "070d5d78902fa837cb16_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "read_file", "question": "Load the file at /notes/setup.txt and read it.", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"service.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "e91695694d804dc7cf13_hi", "language": "hi", "backend": "supabase", "operation": "ticket", "question": "टिकट TKT-9915 का डेटा फेच करें।", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-9915"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9915\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "5e59d3140947b3c7bc4a_hinglish", "language": "hinglish", "backend": "mysql", "operation": "weather", "question": "Jaipur ka mausam fahrenheit mein lao.", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Jaipur", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "5e59d3140947b3c7bc4a_en", "language": "en", "backend": "mysql", "operation": "weather", "question": "Display the weather for Jaipur using fahrenheit.", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Jaipur", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "5e59d3140947b3c7bc4a_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "weather", "question": "Please show me weather for Jaipur in fahrenheit.", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Jaipur", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "37a4acee442f2b6667ce_en", "language": "en", "backend": "mysql", "operation": "search", "question": "Look for API authentication inside the documentation.", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "API authentication"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"API authentication\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "37a4acee442f2b6667ce_hinglish", "language": "hinglish", "backend": "mysql", "operation": "search", "question": "Docs ke andar API authentication dhoondo.", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "API authentication"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"API authentication\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "63e80f5e293fa639c0d9_hinglish", "language": "hinglish", "backend": "mysql", "operation": "weather", "question": "Mumbai ka mausam fahrenheit mein lao.", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Mumbai", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Mumbai\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "ff9bdd2eeb35e58e67f2_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "ticket", "question": "display the record for TKT-370", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-370"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-370\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "132914c1610a3ceada4c_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "ticket", "question": "display the record for TKT-8718", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-8718"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-8718\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "c226064697cb1dc20646_hinglish", "language": "hinglish", "backend": "mysql", "operation": "weather", "question": "Lucknow ka mausam celsius mein lao.", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Lucknow", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "337ec26cf10efe903fb1_hinglish", "language": "hinglish", "backend": "mysql", "operation": "weather", "question": "Delhi ka mausam celsius mein lao.", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Delhi", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "63e80f5e293fa639c0d9_en", "language": "en", "backend": "mysql", "operation": "weather", "question": "Display the weather for Mumbai using fahrenheit.", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Mumbai", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Mumbai\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "48a316f4c32067e3ca40_hi", "language": "hi", "backend": "supabase", "operation": "search", "question": "मुझे दस्तावेज़ीकरण में row level security की आवश्यकता है।", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "row level security"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "b1f06f7bdfe02dd430e7_hinglish", "language": "hinglish", "backend": "mysql", "operation": "weather", "question": "Delhi ka mausam fahrenheit mein lao.", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "2938bd358ea4809d346f_en", "language": "en", "backend": "mysql", "operation": "search", "question": "Look for database backups inside the documentation.", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "2938bd358ea4809d346f_hinglish", "language": "hinglish", "backend": "mysql", "operation": "search", "question": "Docs ke andar database backups dhoondo.", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "c226064697cb1dc20646_en", "language": "en", "backend": "mysql", "operation": "weather", "question": "Display the weather for Lucknow using celsius.", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Lucknow", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "337ec26cf10efe903fb1_en", "language": "en", "backend": "mysql", "operation": "weather", "question": "Display the weather for Delhi using celsius.", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Delhi", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "37a4acee442f2b6667ce_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "search", "question": "help me find API authentication in the documentation", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "API authentication"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"API authentication\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "63e80f5e293fa639c0d9_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "weather", "question": "Please show me weather for Mumbai in fahrenheit.", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Mumbai", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Mumbai\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "b1f06f7bdfe02dd430e7_en", "language": "en", "backend": "mysql", "operation": "weather", "question": "Display the weather for Delhi using fahrenheit.", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "ff9bdd2eeb35e58e67f2_hinglish", "language": "hinglish", "backend": "mysql", "operation": "ticket", "question": "ticket TKT-370 ki info retrieve karo.", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-370"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-370\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "132914c1610a3ceada4c_hinglish", "language": "hinglish", "backend": "mysql", "operation": "ticket", "question": "ticket TKT-8718 ki info retrieve karo.", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-8718"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-8718\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "c226064697cb1dc20646_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "weather", "question": "Please show me weather for Lucknow in celsius.", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Lucknow", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "eee82f0e2886d3c03463_en", "language": "en", "backend": "mysql", "operation": "search", "question": "Look for row level security inside the documentation.", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "row level security"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "eee82f0e2886d3c03463_hinglish", "language": "hinglish", "backend": "mysql", "operation": "search", "question": "Docs ke andar row level security dhoondo.", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "row level security"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "337ec26cf10efe903fb1_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "weather", "question": "Please show me weather for Delhi in celsius.", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Delhi", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "b1f06f7bdfe02dd430e7_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "weather", "question": "Please show me weather for Delhi in fahrenheit.", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "2938bd358ea4809d346f_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "search", "question": "help me find database backups in the documentation", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "9c2fcd765bf5c2df33aa_hi", "language": "hi", "backend": "supabase", "operation": "search", "question": "मुझे दस्तावेज़ीकरण में row level security की आवश्यकता है।", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "row level security"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "e815e61cd1cbba2d7eec_hi", "language": "hi", "backend": "supabase", "operation": "read_file", "question": "/reports/sales.csv पर स्थित फ़ाइल का सामग्री लोड करें।", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/reports/sales.csv"}}, "output": "{\"action\":\"call\",\"name\":\"service.read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "5e59d3140947b3c7bc4a_hi", "language": "hi", "backend": "mysql", "operation": "weather", "question": "Jaipur का मौसम fahrenheit में लाएं।", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Jaipur", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "a9cd5b7c451419ebb9c3_hi", "language": "hi", "backend": "supabase", "operation": "read_file", "question": "/reports/sales.csv पर स्थित फ़ाइल का सामग्री लोड करें।", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/reports/sales.csv"}}, "output": "{\"action\":\"call\",\"name\":\"service.read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "5a26587ff6051e365791_hinglish", "language": "hinglish", "backend": "mysql", "operation": "read_file", "question": "/notes/setup.txt par file load karo.", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"service.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "ff9bdd2eeb35e58e67f2_en", "language": "en", "backend": "mysql", "operation": "ticket", "question": "Please locate the information for support ticket TKT-370.", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-370"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-370\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "132914c1610a3ceada4c_en", "language": "en", "backend": "mysql", "operation": "ticket", "question": "Please locate the information for support ticket TKT-8718.", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-8718"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-8718\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "eee82f0e2886d3c03463_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "search", "question": "help me find row level security in the documentation", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "row level security"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "070d5d78902fa837cb16_hi", "language": "hi", "backend": "supabase", "operation": "read_file", "question": "/notes/setup.txt पर स्थित फ़ाइल का सामग्री लोड करें।", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"service.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "63e80f5e293fa639c0d9_hi", "language": "hi", "backend": "mysql", "operation": "weather", "question": "Mumbai का मौसम fahrenheit में लाएं।", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Mumbai", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Mumbai\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "c226064697cb1dc20646_hi", "language": "hi", "backend": "mysql", "operation": "weather", "question": "Lucknow का मौसम celsius में लाएं।", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Lucknow", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "337ec26cf10efe903fb1_hi", "language": "hi", "backend": "mysql", "operation": "weather", "question": "Delhi का मौसम celsius में लाएं।", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Delhi", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "b1f06f7bdfe02dd430e7_hi", "language": "hi", "backend": "mysql", "operation": "weather", "question": "Delhi का मौसम fahrenheit में लाएं।", "expected": {"action": "call", "name": "service.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "5a26587ff6051e365791_en", "language": "en", "backend": "mysql", "operation": "read_file", "question": "Load the content of the file at /notes/setup.txt.", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"service.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "5a26587ff6051e365791_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "read_file", "question": "Load the file at /notes/setup.txt and read it.", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"service.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "ff9bdd2eeb35e58e67f2_hi", "language": "hi", "backend": "mysql", "operation": "ticket", "question": "टिकट TKT-370 का डेटा फेच करें।", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-370"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-370\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "132914c1610a3ceada4c_hi", "language": "hi", "backend": "mysql", "operation": "ticket", "question": "टिकट TKT-8718 का डेटा फेच करें।", "expected": {"action": "call", "name": "service.get_ticket", "arguments": {"ticket_id": "TKT-8718"}}, "output": "{\"action\":\"call\",\"name\":\"service.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-8718\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "37a4acee442f2b6667ce_hi", "language": "hi", "backend": "mysql", "operation": "search", "question": "मुझे दस्तावेज़ीकरण में API authentication की आवश्यकता है।", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "API authentication"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"API authentication\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "2938bd358ea4809d346f_hi", "language": "hi", "backend": "mysql", "operation": "search", "question": "मुझे दस्तावेज़ीकरण में database backups की आवश्यकता है।", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "eee82f0e2886d3c03463_hi", "language": "hi", "backend": "mysql", "operation": "search", "question": "मुझे दस्तावेज़ीकरण में row level security की आवश्यकता है।", "expected": {"action": "call", "name": "service.search_docs", "arguments": {"query": "row level security"}}, "output": "{\"action\":\"call\",\"name\":\"service.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
{"id": "5a26587ff6051e365791_hi", "language": "hi", "backend": "mysql", "operation": "read_file", "question": "/notes/setup.txt पर स्थित फ़ाइल का सामग्री लोड करें।", "expected": {"action": "call", "name": "service.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"service.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}