| { | |
| "model": "t5-base", | |
| "dataset": "bird", | |
| "run": 1, | |
| "checkpoint": "bird/t5-base/run_1/best", | |
| "eval_beams": 4, | |
| "n_samples": 1534, | |
| "em_raw": 0.0339, | |
| "em_norm": 0.0346, | |
| "ex": 0.0, | |
| "ex_errors": 0, | |
| "ex_missing_db": 1534, | |
| "by_difficulty": { | |
| "simple": { | |
| "total": 925, | |
| "em_norm": 0.0551, | |
| "ex": 0.0 | |
| }, | |
| "moderate": { | |
| "total": 464, | |
| "em_norm": 0.0022, | |
| "ex": 0.0 | |
| }, | |
| "challenging": { | |
| "total": 145, | |
| "em_norm": 0.0069, | |
| "ex": 0.0 | |
| } | |
| }, | |
| "by_sql_type": { | |
| "ORDER_BY": { | |
| "total": 305, | |
| "em_norm": 0.023, | |
| "ex": 0.0 | |
| }, | |
| "LIMIT": { | |
| "total": 293, | |
| "em_norm": 0.0239, | |
| "ex": 0.0 | |
| }, | |
| "JOIN": { | |
| "total": 1140, | |
| "em_norm": 0.0202, | |
| "ex": 0.0 | |
| }, | |
| "AGGREGATION": { | |
| "total": 659, | |
| "em_norm": 0.0212, | |
| "ex": 0.0 | |
| }, | |
| "NESTED": { | |
| "total": 118, | |
| "em_norm": 0.0508, | |
| "ex": 0.0 | |
| }, | |
| "GROUP_BY": { | |
| "total": 120, | |
| "em_norm": 0.0, | |
| "ex": 0.0 | |
| }, | |
| "HAVING": { | |
| "total": 21, | |
| "em_norm": 0.0, | |
| "ex": 0.0 | |
| }, | |
| "SIMPLE": { | |
| "total": 119, | |
| "em_norm": 0.1176, | |
| "ex": 0.0 | |
| } | |
| }, | |
| "error_types": { | |
| "EXTRA_JOIN": 200, | |
| "WRONG_TABLE": 676, | |
| "MISSING_ORDER_BY": 24, | |
| "OTHER": 450, | |
| "MISSING_WHERE": 18, | |
| "MISSING_GROUP_BY": 33, | |
| "MISSING_JOIN": 80 | |
| } | |
| } |