| { | |
| "model": "t5-base", | |
| "dataset": "spider", | |
| "run": 3, | |
| "eval_beams": 4, | |
| "n_samples": 1034, | |
| "em_raw": 0.3337, | |
| "em_norm": 0.3337, | |
| "ex": 0.4333, | |
| "ex_errors": 358, | |
| "by_sql_type": { | |
| "AGGREGATION": { | |
| "total": 551, | |
| "em_norm": 0.3485, | |
| "ex": 0.4319 | |
| }, | |
| "ORDER_BY": { | |
| "total": 237, | |
| "em_norm": 0.3165, | |
| "ex": 0.4388 | |
| }, | |
| "LIMIT": { | |
| "total": 189, | |
| "em_norm": 0.2698, | |
| "ex": 0.4021 | |
| }, | |
| "SIMPLE": { | |
| "total": 185, | |
| "em_norm": 0.4919, | |
| "ex": 0.6486 | |
| }, | |
| "GROUP_BY": { | |
| "total": 277, | |
| "em_norm": 0.2599, | |
| "ex": 0.3357 | |
| }, | |
| "NESTED": { | |
| "total": 83, | |
| "em_norm": 0.3012, | |
| "ex": 0.3012 | |
| }, | |
| "JOIN": { | |
| "total": 408, | |
| "em_norm": 0.027, | |
| "ex": 0.1324 | |
| }, | |
| "HAVING": { | |
| "total": 79, | |
| "em_norm": 0.1772, | |
| "ex": 0.2152 | |
| } | |
| }, | |
| "by_hardness": { | |
| "easy": { | |
| "total": 456, | |
| "em_norm": 0.5285, | |
| "ex": 0.6535 | |
| }, | |
| "medium": { | |
| "total": 402, | |
| "em_norm": 0.1617, | |
| "ex": 0.2687 | |
| }, | |
| "hard": { | |
| "total": 168, | |
| "em_norm": 0.2321, | |
| "ex": 0.25 | |
| }, | |
| "extra_hard": { | |
| "total": 8, | |
| "em_norm": 0.0, | |
| "ex": 0.0 | |
| } | |
| }, | |
| "error_types": { | |
| "OTHER": 176, | |
| "WRONG_COLUMNS": 115, | |
| "MISSING_JOIN": 107, | |
| "WRONG_TABLE": 243, | |
| "MISSING_GROUP_BY": 16, | |
| "EXTRA_JOIN": 22, | |
| "MISSING_ORDER_BY": 8, | |
| "MISSING_WHERE": 2 | |
| } | |
| } |