| { | |
| "model": "t5-base", | |
| "dataset": "spider", | |
| "run": 1, | |
| "eval_beams": 4, | |
| "n_samples": 1034, | |
| "em_raw": 0.3221, | |
| "em_norm": 0.3781, | |
| "ex": 0.472, | |
| "ex_errors": 372, | |
| "by_sql_type": { | |
| "AGGREGATION": { | |
| "total": 551, | |
| "em_norm": 0.3902, | |
| "ex": 0.4537 | |
| }, | |
| "ORDER_BY": { | |
| "total": 237, | |
| "em_norm": 0.3544, | |
| "ex": 0.4895 | |
| }, | |
| "LIMIT": { | |
| "total": 189, | |
| "em_norm": 0.291, | |
| "ex": 0.4233 | |
| }, | |
| "SIMPLE": { | |
| "total": 185, | |
| "em_norm": 0.5297, | |
| "ex": 0.6216 | |
| }, | |
| "GROUP_BY": { | |
| "total": 277, | |
| "em_norm": 0.3141, | |
| "ex": 0.4116 | |
| }, | |
| "NESTED": { | |
| "total": 83, | |
| "em_norm": 0.3012, | |
| "ex": 0.3373 | |
| }, | |
| "JOIN": { | |
| "total": 408, | |
| "em_norm": 0.1054, | |
| "ex": 0.2574 | |
| }, | |
| "HAVING": { | |
| "total": 79, | |
| "em_norm": 0.2658, | |
| "ex": 0.3544 | |
| } | |
| }, | |
| "by_hardness": { | |
| "easy": { | |
| "total": 456, | |
| "em_norm": 0.5658, | |
| "ex": 0.6272 | |
| }, | |
| "medium": { | |
| "total": 402, | |
| "em_norm": 0.2164, | |
| "ex": 0.3657 | |
| }, | |
| "hard": { | |
| "total": 168, | |
| "em_norm": 0.2738, | |
| "ex": 0.3214 | |
| }, | |
| "extra_hard": { | |
| "total": 8, | |
| "em_norm": 0.0, | |
| "ex": 0.125 | |
| } | |
| }, | |
| "error_types": { | |
| "OTHER": 165, | |
| "WRONG_COLUMNS": 125, | |
| "WRONG_TABLE": 255, | |
| "EXTRA_JOIN": 42, | |
| "MISSING_JOIN": 42, | |
| "MISSING_ORDER_BY": 7, | |
| "MISSING_GROUP_BY": 6, | |
| "MISSING_WHERE": 1 | |
| } | |
| } |