{
  "model": "t5-base",
  "dataset": "spider",
  "run": 3,
  "eval_beams": 4,
  "n_samples": 1034,
  "em_raw": 0.3337,
  "em_norm": 0.3337,
  "ex": 0.4333,
  "ex_errors": 358,
  "by_sql_type": {
    "AGGREGATION": {
      "total": 551,
      "em_norm": 0.3485,
      "ex": 0.4319
    },
    "ORDER_BY": {
      "total": 237,
      "em_norm": 0.3165,
      "ex": 0.4388
    },
    "LIMIT": {
      "total": 189,
      "em_norm": 0.2698,
      "ex": 0.4021
    },
    "SIMPLE": {
      "total": 185,
      "em_norm": 0.4919,
      "ex": 0.6486
    },
    "GROUP_BY": {
      "total": 277,
      "em_norm": 0.2599,
      "ex": 0.3357
    },
    "NESTED": {
      "total": 83,
      "em_norm": 0.3012,
      "ex": 0.3012
    },
    "JOIN": {
      "total": 408,
      "em_norm": 0.027,
      "ex": 0.1324
    },
    "HAVING": {
      "total": 79,
      "em_norm": 0.1772,
      "ex": 0.2152
    }
  },
  "by_hardness": {
    "easy": {
      "total": 456,
      "em_norm": 0.5285,
      "ex": 0.6535
    },
    "medium": {
      "total": 402,
      "em_norm": 0.1617,
      "ex": 0.2687
    },
    "hard": {
      "total": 168,
      "em_norm": 0.2321,
      "ex": 0.25
    },
    "extra_hard": {
      "total": 8,
      "em_norm": 0.0,
      "ex": 0.0
    }
  },
  "error_types": {
    "OTHER": 176,
    "WRONG_COLUMNS": 115,
    "MISSING_JOIN": 107,
    "WRONG_TABLE": 243,
    "MISSING_GROUP_BY": 16,
    "EXTRA_JOIN": 22,
    "MISSING_ORDER_BY": 8,
    "MISSING_WHERE": 2
  }
}