| { |
| "examples": 1196, |
| "seconds": 17.66167116165161, |
| "decoding": "raw greedy, no repair, no teacher fallback", |
| "checkpoint_sha256": "9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692", |
| "sql_metric": "Compile against the supplied schema, then compare results on two generated SQLite fixtures after dialect adaptation; not native MySQL/PostgreSQL execution.", |
| "metrics": { |
| "json_valid": { |
| "correct": 1196, |
| "total": 1196, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 1191, |
| "total": 1196, |
| "rate": 0.995819397993311 |
| }, |
| "action_correct": { |
| "correct": 1192, |
| "total": 1196, |
| "rate": 0.9966555183946488 |
| }, |
| "tool_correct": { |
| "correct": 1088, |
| "total": 1100, |
| "rate": 0.9890909090909091 |
| }, |
| "arguments_exact": { |
| "correct": 1014, |
| "total": 1100, |
| "rate": 0.9218181818181819 |
| }, |
| "sql_equivalent": { |
| "correct": 896, |
| "total": 912, |
| "rate": 0.9824561403508771 |
| }, |
| "success": { |
| "correct": 1106, |
| "total": 1196, |
| "rate": 0.9247491638795987 |
| } |
| }, |
| "groups": { |
| "language:noisy_en": { |
| "json_valid": { |
| "correct": 299, |
| "total": 299, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 298, |
| "total": 299, |
| "rate": 0.9966555183946488 |
| }, |
| "action_correct": { |
| "correct": 299, |
| "total": 299, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 274, |
| "total": 275, |
| "rate": 0.9963636363636363 |
| }, |
| "arguments_exact": { |
| "correct": 252, |
| "total": 275, |
| "rate": 0.9163636363636364 |
| }, |
| "sql_equivalent": { |
| "correct": 228, |
| "total": 228, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 276, |
| "total": 299, |
| "rate": 0.9230769230769231 |
| } |
| }, |
| "backend:supabase": { |
| "json_valid": { |
| "correct": 588, |
| "total": 588, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 588, |
| "total": 588, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 588, |
| "total": 588, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 541, |
| "total": 544, |
| "rate": 0.9944852941176471 |
| }, |
| "arguments_exact": { |
| "correct": 480, |
| "total": 544, |
| "rate": 0.8823529411764706 |
| }, |
| "sql_equivalent": { |
| "correct": 429, |
| "total": 436, |
| "rate": 0.9839449541284404 |
| }, |
| "success": { |
| "correct": 524, |
| "total": 588, |
| "rate": 0.891156462585034 |
| } |
| }, |
| "operation:missing_schema": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 20, |
| "total": 24, |
| "rate": 0.8333333333333334 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 20, |
| "total": 24, |
| "rate": 0.8333333333333334 |
| } |
| }, |
| "language:en": { |
| "json_valid": { |
| "correct": 299, |
| "total": 299, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 298, |
| "total": 299, |
| "rate": 0.9966555183946488 |
| }, |
| "action_correct": { |
| "correct": 297, |
| "total": 299, |
| "rate": 0.9933110367892977 |
| }, |
| "tool_correct": { |
| "correct": 274, |
| "total": 275, |
| "rate": 0.9963636363636363 |
| }, |
| "arguments_exact": { |
| "correct": 246, |
| "total": 275, |
| "rate": 0.8945454545454545 |
| }, |
| "sql_equivalent": { |
| "correct": 222, |
| "total": 228, |
| "rate": 0.9736842105263158 |
| }, |
| "success": { |
| "correct": 268, |
| "total": 299, |
| "rate": 0.8963210702341137 |
| } |
| }, |
| "language:hinglish": { |
| "json_valid": { |
| "correct": 299, |
| "total": 299, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 297, |
| "total": 299, |
| "rate": 0.9933110367892977 |
| }, |
| "action_correct": { |
| "correct": 297, |
| "total": 299, |
| "rate": 0.9933110367892977 |
| }, |
| "tool_correct": { |
| "correct": 273, |
| "total": 275, |
| "rate": 0.9927272727272727 |
| }, |
| "arguments_exact": { |
| "correct": 262, |
| "total": 275, |
| "rate": 0.9527272727272728 |
| }, |
| "sql_equivalent": { |
| "correct": 228, |
| "total": 228, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 284, |
| "total": 299, |
| "rate": 0.9498327759197325 |
| } |
| }, |
| "operation:sql_error": { |
| "json_valid": { |
| "correct": 20, |
| "total": 20, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 20, |
| "total": 20, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 20, |
| "total": 20, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 20, |
| "total": 20, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 12, |
| "total": 20, |
| "rate": 0.6 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 12, |
| "total": 20, |
| "rate": 0.6 |
| } |
| }, |
| "backend:mysql": { |
| "json_valid": { |
| "correct": 608, |
| "total": 608, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 603, |
| "total": 608, |
| "rate": 0.9917763157894737 |
| }, |
| "action_correct": { |
| "correct": 604, |
| "total": 608, |
| "rate": 0.993421052631579 |
| }, |
| "tool_correct": { |
| "correct": 547, |
| "total": 556, |
| "rate": 0.9838129496402878 |
| }, |
| "arguments_exact": { |
| "correct": 534, |
| "total": 556, |
| "rate": 0.960431654676259 |
| }, |
| "sql_equivalent": { |
| "correct": 467, |
| "total": 476, |
| "rate": 0.9810924369747899 |
| }, |
| "success": { |
| "correct": 582, |
| "total": 608, |
| "rate": 0.9572368421052632 |
| } |
| }, |
| "operation:list_tables": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 12, |
| "total": 24, |
| "rate": 0.5 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 12, |
| "total": 24, |
| "rate": 0.5 |
| } |
| }, |
| "operation:all": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:count": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 23, |
| "total": 28, |
| "rate": 0.8214285714285714 |
| }, |
| "sql_equivalent": { |
| "correct": 23, |
| "total": 28, |
| "rate": 0.8214285714285714 |
| }, |
| "success": { |
| "correct": 23, |
| "total": 28, |
| "rate": 0.8214285714285714 |
| } |
| }, |
| "operation:avg": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| } |
| }, |
| "operation:describe": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 23, |
| "total": 24, |
| "rate": 0.9583333333333334 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 23, |
| "total": 24, |
| "rate": 0.9583333333333334 |
| }, |
| "arguments_exact": { |
| "correct": 19, |
| "total": 24, |
| "rate": 0.7916666666666666 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 19, |
| "total": 24, |
| "rate": 0.7916666666666666 |
| } |
| }, |
| "operation:project": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:gte": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| } |
| }, |
| "operation:min": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 28, |
| "rate": 0.8571428571428571 |
| }, |
| "sql_equivalent": { |
| "correct": 24, |
| "total": 28, |
| "rate": 0.8571428571428571 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 28, |
| "rate": 0.8571428571428571 |
| } |
| }, |
| "operation:null": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| } |
| }, |
| "operation:write": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 20, |
| "total": 24, |
| "rate": 0.8333333333333334 |
| }, |
| "tool_correct": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "arguments_exact": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 20, |
| "total": 24, |
| "rate": 0.8333333333333334 |
| } |
| }, |
| "operation:or": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:count_eq": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 25, |
| "total": 28, |
| "rate": 0.8928571428571429 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 25, |
| "total": 28, |
| "rate": 0.8928571428571429 |
| } |
| }, |
| "operation:contains": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:sum": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| } |
| }, |
| "operation:sort_desc": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| } |
| }, |
| "operation:eq": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:project_eq": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "language:hi": { |
| "json_valid": { |
| "correct": 299, |
| "total": 299, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 298, |
| "total": 299, |
| "rate": 0.9966555183946488 |
| }, |
| "action_correct": { |
| "correct": 299, |
| "total": 299, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 267, |
| "total": 275, |
| "rate": 0.9709090909090909 |
| }, |
| "arguments_exact": { |
| "correct": 254, |
| "total": 275, |
| "rate": 0.9236363636363636 |
| }, |
| "sql_equivalent": { |
| "correct": 218, |
| "total": 228, |
| "rate": 0.956140350877193 |
| }, |
| "success": { |
| "correct": 278, |
| "total": 299, |
| "rate": 0.9297658862876255 |
| } |
| }, |
| "operation:year": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| } |
| }, |
| "operation:bottom": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:distinct": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:ambiguous": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "arguments_exact": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:max": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| } |
| }, |
| "operation:group_count": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:sort_asc": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:not_null": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| } |
| }, |
| "operation:lte": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:lt": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| } |
| }, |
| "operation:group_sum": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:between": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 21, |
| "total": 28, |
| "rate": 0.75 |
| }, |
| "arguments_exact": { |
| "correct": 21, |
| "total": 28, |
| "rate": 0.75 |
| }, |
| "sql_equivalent": { |
| "correct": 21, |
| "total": 28, |
| "rate": 0.75 |
| }, |
| "success": { |
| "correct": 21, |
| "total": 28, |
| "rate": 0.75 |
| } |
| }, |
| "operation:month": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 28, |
| "rate": 0.8571428571428571 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 28, |
| "rate": 0.8571428571428571 |
| } |
| }, |
| "operation:and": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| } |
| }, |
| "operation:having": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 20, |
| "total": 28, |
| "rate": 0.7142857142857143 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 20, |
| "total": 28, |
| "rate": 0.7142857142857143 |
| } |
| }, |
| "operation:missing_value": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "arguments_exact": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:final": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "arguments_exact": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:date_after": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:gt": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 26, |
| "total": 28, |
| "rate": 0.9285714285714286 |
| } |
| }, |
| "operation:top": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:join_filter": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:join": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:weather": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:search": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 20, |
| "total": 24, |
| "rate": 0.8333333333333334 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 20, |
| "total": 24, |
| "rate": 0.8333333333333334 |
| }, |
| "arguments_exact": { |
| "correct": 20, |
| "total": 24, |
| "rate": 0.8333333333333334 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 20, |
| "total": 24, |
| "rate": 0.8333333333333334 |
| } |
| }, |
| "operation:ticket": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:read_file": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| } |
| } |
| } |
