{
  "model": "t5-base",
  "dataset": "bird",
  "run": 1,
  "checkpoint": "bird/t5-base/run_1/best",
  "eval_beams": 4,
  "n_samples": 1534,
  "em_raw": 0.0339,
  "em_norm": 0.0346,
  "ex": 0.0,
  "ex_errors": 0,
  "ex_missing_db": 1534,
  "by_difficulty": {
    "simple": {
      "total": 925,
      "em_norm": 0.0551,
      "ex": 0.0
    },
    "moderate": {
      "total": 464,
      "em_norm": 0.0022,
      "ex": 0.0
    },
    "challenging": {
      "total": 145,
      "em_norm": 0.0069,
      "ex": 0.0
    }
  },
  "by_sql_type": {
    "ORDER_BY": {
      "total": 305,
      "em_norm": 0.023,
      "ex": 0.0
    },
    "LIMIT": {
      "total": 293,
      "em_norm": 0.0239,
      "ex": 0.0
    },
    "JOIN": {
      "total": 1140,
      "em_norm": 0.0202,
      "ex": 0.0
    },
    "AGGREGATION": {
      "total": 659,
      "em_norm": 0.0212,
      "ex": 0.0
    },
    "NESTED": {
      "total": 118,
      "em_norm": 0.0508,
      "ex": 0.0
    },
    "GROUP_BY": {
      "total": 120,
      "em_norm": 0.0,
      "ex": 0.0
    },
    "HAVING": {
      "total": 21,
      "em_norm": 0.0,
      "ex": 0.0
    },
    "SIMPLE": {
      "total": 119,
      "em_norm": 0.1176,
      "ex": 0.0
    }
  },
  "error_types": {
    "EXTRA_JOIN": 200,
    "WRONG_TABLE": 676,
    "MISSING_ORDER_BY": 24,
    "OTHER": 450,
    "MISSING_WHERE": 18,
    "MISSING_GROUP_BY": 33,
    "MISSING_JOIN": 80
  }
}