| { |
| "examples": 1200, |
| "seconds": 33.96494126319885, |
| "decoding": "raw greedy, no repair, no teacher fallback", |
| "checkpoint_sha256": "9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692", |
| "sql_metric": "Compile against the supplied schema, then compare results on two generated SQLite fixtures after dialect adaptation; not native MySQL/PostgreSQL execution.", |
| "metrics": { |
| "json_valid": { |
| "correct": 1200, |
| "total": 1200, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 1199, |
| "total": 1200, |
| "rate": 0.9991666666666666 |
| }, |
| "action_correct": { |
| "correct": 1193, |
| "total": 1200, |
| "rate": 0.9941666666666666 |
| }, |
| "tool_correct": { |
| "correct": 1103, |
| "total": 1104, |
| "rate": 0.9990942028985508 |
| }, |
| "arguments_exact": { |
| "correct": 1085, |
| "total": 1104, |
| "rate": 0.9827898550724637 |
| }, |
| "sql_equivalent": { |
| "correct": 899, |
| "total": 912, |
| "rate": 0.9857456140350878 |
| }, |
| "success": { |
| "correct": 1174, |
| "total": 1200, |
| "rate": 0.9783333333333334 |
| } |
| }, |
| "groups": { |
| "language:noisy_en": { |
| "json_valid": { |
| "correct": 300, |
| "total": 300, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 300, |
| "total": 300, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 300, |
| "total": 300, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 276, |
| "total": 276, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 275, |
| "total": 276, |
| "rate": 0.9963768115942029 |
| }, |
| "sql_equivalent": { |
| "correct": 227, |
| "total": 228, |
| "rate": 0.9956140350877193 |
| }, |
| "success": { |
| "correct": 299, |
| "total": 300, |
| "rate": 0.9966666666666667 |
| } |
| }, |
| "backend:supabase": { |
| "json_valid": { |
| "correct": 604, |
| "total": 604, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 603, |
| "total": 604, |
| "rate": 0.9983443708609272 |
| }, |
| "action_correct": { |
| "correct": 601, |
| "total": 604, |
| "rate": 0.9950331125827815 |
| }, |
| "tool_correct": { |
| "correct": 559, |
| "total": 560, |
| "rate": 0.9982142857142857 |
| }, |
| "arguments_exact": { |
| "correct": 551, |
| "total": 560, |
| "rate": 0.9839285714285714 |
| }, |
| "sql_equivalent": { |
| "correct": 461, |
| "total": 468, |
| "rate": 0.9850427350427351 |
| }, |
| "success": { |
| "correct": 592, |
| "total": 604, |
| "rate": 0.9801324503311258 |
| } |
| }, |
| "operation:missing_schema": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "language:en": { |
| "json_valid": { |
| "correct": 300, |
| "total": 300, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 300, |
| "total": 300, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 294, |
| "total": 300, |
| "rate": 0.98 |
| }, |
| "tool_correct": { |
| "correct": 276, |
| "total": 276, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 268, |
| "total": 276, |
| "rate": 0.9710144927536232 |
| }, |
| "sql_equivalent": { |
| "correct": 222, |
| "total": 228, |
| "rate": 0.9736842105263158 |
| }, |
| "success": { |
| "correct": 286, |
| "total": 300, |
| "rate": 0.9533333333333334 |
| } |
| }, |
| "language:hinglish": { |
| "json_valid": { |
| "correct": 300, |
| "total": 300, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 299, |
| "total": 300, |
| "rate": 0.9966666666666667 |
| }, |
| "action_correct": { |
| "correct": 300, |
| "total": 300, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 275, |
| "total": 276, |
| "rate": 0.9963768115942029 |
| }, |
| "arguments_exact": { |
| "correct": 268, |
| "total": 276, |
| "rate": 0.9710144927536232 |
| }, |
| "sql_equivalent": { |
| "correct": 223, |
| "total": 228, |
| "rate": 0.9780701754385965 |
| }, |
| "success": { |
| "correct": 292, |
| "total": 300, |
| "rate": 0.9733333333333334 |
| } |
| }, |
| "backend:mysql": { |
| "json_valid": { |
| "correct": 596, |
| "total": 596, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 596, |
| "total": 596, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 592, |
| "total": 596, |
| "rate": 0.9932885906040269 |
| }, |
| "tool_correct": { |
| "correct": 544, |
| "total": 544, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 534, |
| "total": 544, |
| "rate": 0.9816176470588235 |
| }, |
| "sql_equivalent": { |
| "correct": 438, |
| "total": 444, |
| "rate": 0.9864864864864865 |
| }, |
| "success": { |
| "correct": 582, |
| "total": 596, |
| "rate": 0.9765100671140939 |
| } |
| }, |
| "operation:all": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:write": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 18, |
| "total": 24, |
| "rate": 0.75 |
| }, |
| "tool_correct": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "arguments_exact": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 18, |
| "total": 24, |
| "rate": 0.75 |
| } |
| }, |
| "operation:distinct": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:max": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:min": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:avg": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:group_count": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 25, |
| "total": 28, |
| "rate": 0.8928571428571429 |
| }, |
| "sql_equivalent": { |
| "correct": 25, |
| "total": 28, |
| "rate": 0.8928571428571429 |
| }, |
| "success": { |
| "correct": 25, |
| "total": 28, |
| "rate": 0.8928571428571429 |
| } |
| }, |
| "language:hi": { |
| "json_valid": { |
| "correct": 300, |
| "total": 300, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 300, |
| "total": 300, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 299, |
| "total": 300, |
| "rate": 0.9966666666666667 |
| }, |
| "tool_correct": { |
| "correct": 276, |
| "total": 276, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 274, |
| "total": 276, |
| "rate": 0.9927536231884058 |
| }, |
| "sql_equivalent": { |
| "correct": 227, |
| "total": 228, |
| "rate": 0.9956140350877193 |
| }, |
| "success": { |
| "correct": 297, |
| "total": 300, |
| "rate": 0.99 |
| } |
| }, |
| "operation:sum": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 27, |
| "total": 28, |
| "rate": 0.9642857142857143 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 27, |
| "total": 28, |
| "rate": 0.9642857142857143 |
| } |
| }, |
| "operation:count_eq": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:count": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 22, |
| "total": 28, |
| "rate": 0.7857142857142857 |
| }, |
| "sql_equivalent": { |
| "correct": 22, |
| "total": 28, |
| "rate": 0.7857142857142857 |
| }, |
| "success": { |
| "correct": 22, |
| "total": 28, |
| "rate": 0.7857142857142857 |
| } |
| }, |
| "operation:sql_error": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 20, |
| "total": 24, |
| "rate": 0.8333333333333334 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 20, |
| "total": 24, |
| "rate": 0.8333333333333334 |
| } |
| }, |
| "operation:project": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 28, |
| "rate": 0.8571428571428571 |
| }, |
| "sql_equivalent": { |
| "correct": 24, |
| "total": 28, |
| "rate": 0.8571428571428571 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 28, |
| "rate": 0.8571428571428571 |
| } |
| }, |
| "operation:lt": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:top": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:group_sum": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:sort_desc": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:ambiguous": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "arguments_exact": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:gte": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:eq": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:or": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:month": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:project_eq": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:contains": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:not_null": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:gt": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:lte": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:bottom": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:having": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:sort_asc": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:null": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:year": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:missing_value": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "arguments_exact": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:and": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:date_after": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:final": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 23, |
| "total": 24, |
| "rate": 0.9583333333333334 |
| }, |
| "tool_correct": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "arguments_exact": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 23, |
| "total": 24, |
| "rate": 0.9583333333333334 |
| } |
| }, |
| "operation:list_tables": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:describe": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:join": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:join_filter": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:between": { |
| "json_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 28, |
| "total": 28, |
| "rate": 1.0 |
| } |
| }, |
| "operation:search": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:weather": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:ticket": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| } |
| }, |
| "operation:read_file": { |
| "json_valid": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 23, |
| "total": 24, |
| "rate": 0.9583333333333334 |
| }, |
| "action_correct": { |
| "correct": 24, |
| "total": 24, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 23, |
| "total": 24, |
| "rate": 0.9583333333333334 |
| }, |
| "arguments_exact": { |
| "correct": 23, |
| "total": 24, |
| "rate": 0.9583333333333334 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 23, |
| "total": 24, |
| "rate": 0.9583333333333334 |
| } |
| } |
| }, |
| "rescoring": "Same saved raw outputs, with explicit supplied-schema compilation; no generation or output repair." |
| } |
