eval_spider_dev_100.json

{
  "base": {
    "by_db": {
      "car_1": {
        "correct": 7,
        "execution_accuracy": 0.5385,
        "total": 13
      },
      "concert_singer": {
        "correct": 32,
        "execution_accuracy": 0.7111,
        "total": 45
      },
      "pets_1": {
        "correct": 2,
        "execution_accuracy": 0.0476,
        "total": 42
      }
    },
    "correct": 41,
    "execution_accuracy": 0.41,
    "top_prediction_errors": {
      "near \"The\": syntax error": 5,
      "near \"Thinking\": syntax error": 49
    },
    "total": 100
  },
  "benchmark": "Spider dev official SQLite execution, first 100 examples",
  "delta": {
    "correct": 10,
    "execution_accuracy": 0.1
  },
  "fine_tuned": {
    "by_db": {
      "car_1": {
        "correct": 5,
        "execution_accuracy": 0.3846,
        "total": 13
      },
      "concert_singer": {
        "correct": 27,
        "execution_accuracy": 0.6,
        "total": 45
      },
      "pets_1": {
        "correct": 19,
        "execution_accuracy": 0.4524,
        "total": 42
      }
    },
    "correct": 51,
    "execution_accuracy": 0.51,
    "top_prediction_errors": {
      "near \"The\": syntax error": 39
    },
    "total": 100
  }
}