{
  "adapter": "sahilsangwan/qwen35-4b-text-to-sql-data-forge-lora",
  "base_model": "Qwen/Qwen3.5-4B",
  "benchmark": "Spider dev",
  "conclusion": "The adapter improves over the local base model on full Spider dev after deterministic SQL extraction, but does not beat the cited public Sonnet-level Spider numbers.",
  "database_dir": "external/spider_download/spider_data/database",
  "dataset_source": "official Spider dev set",
  "dialect": "SQLite",
  "generation": {
    "max_tokens": 256,
    "temperature": 0.0
  },
  "public_reference_numbers": [
    {
      "benchmark": "Spider test",
      "execution_accuracy": 0.6991,
      "model": "Claude 3.5 Sonnet",
      "note": "Different split/harness; included only as public reference.",
      "source": "Optimizing Reasoning for Text-to-SQL with Execution Feedback, ACL Findings 2025"
    },
    {
      "benchmark": "Spider",
      "execution_accuracy": 0.857,
      "model": "Claude 3.5 Sonnet",
      "note": "Different harness; included only as public reference.",
      "source": "Dataworkers text-to-SQL benchmark article"
    }
  ],
  "raw_execution": {
    "base": {
      "correct": 428,
      "execution_accuracy": 0.4139
    },
    "delta_points": -4.06,
    "fine_tuned": {
      "correct": 386,
      "execution_accuracy": 0.3733
    },
    "note": "Raw model output was evaluated without SQL extraction; many generations contained reasoning text before SQL."
  },
  "sample_count": 1034,
  "sql_extracted_execution": {
    "base": {
      "correct": 422,
      "execution_accuracy": 0.4081,
      "total": 1034,
      "valid_sql": 573,
      "valid_sql_rate": 0.5542
    },
    "base_extraction": {
      "changed": 840,
      "input": "generation/niches/text-to-sql/evals/spider_dev_qwen35_lora_800/base_predictions_full.jsonl",
      "no_sql_found": 194,
      "out": "generation/niches/text-to-sql/evals/spider_dev_qwen35_lora_800/base_predictions_full_sql_extracted.jsonl",
      "total": 1034,
      "unchanged": 0
    },
    "execution_accuracy_delta_points": 17.02,
    "fine_tuned": {
      "correct": 598,
      "execution_accuracy": 0.5783,
      "total": 1034,
      "valid_sql": 809,
      "valid_sql_rate": 0.7824
    },
    "fine_tuned_extraction": {
      "changed": 1014,
      "input": "generation/niches/text-to-sql/evals/spider_dev_qwen35_lora_800/fine_tuned_predictions_full.jsonl",
      "no_sql_found": 20,
      "out": "generation/niches/text-to-sql/evals/spider_dev_qwen35_lora_800/fine_tuned_predictions_full_sql_extracted.jsonl",
      "total": 1034,
      "unchanged": 0
    },
    "valid_sql_delta_points": 22.82
  }
}