| { |
| "adapter": "sahilsangwan/qwen35-4b-text-to-sql-data-forge-lora", |
| "base_model": "Qwen/Qwen3.5-4B", |
| "benchmark": "Spider dev", |
| "conclusion": "The adapter improves over the local base model on full Spider dev after deterministic SQL extraction, but does not beat the cited public Sonnet-level Spider numbers.", |
| "database_dir": "external/spider_download/spider_data/database", |
| "dataset_source": "official Spider dev set", |
| "dialect": "SQLite", |
| "generation": { |
| "max_tokens": 256, |
| "temperature": 0.0 |
| }, |
| "public_reference_numbers": [ |
| { |
| "benchmark": "Spider test", |
| "execution_accuracy": 0.6991, |
| "model": "Claude 3.5 Sonnet", |
| "note": "Different split/harness; included only as public reference.", |
| "source": "Optimizing Reasoning for Text-to-SQL with Execution Feedback, ACL Findings 2025" |
| }, |
| { |
| "benchmark": "Spider", |
| "execution_accuracy": 0.857, |
| "model": "Claude 3.5 Sonnet", |
| "note": "Different harness; included only as public reference.", |
| "source": "Dataworkers text-to-SQL benchmark article" |
| } |
| ], |
| "raw_execution": { |
| "base": { |
| "correct": 428, |
| "execution_accuracy": 0.4139 |
| }, |
| "delta_points": -4.06, |
| "fine_tuned": { |
| "correct": 386, |
| "execution_accuracy": 0.3733 |
| }, |
| "note": "Raw model output was evaluated without SQL extraction; many generations contained reasoning text before SQL." |
| }, |
| "sample_count": 1034, |
| "sql_extracted_execution": { |
| "base": { |
| "correct": 422, |
| "execution_accuracy": 0.4081, |
| "total": 1034, |
| "valid_sql": 573, |
| "valid_sql_rate": 0.5542 |
| }, |
| "base_extraction": { |
| "changed": 840, |
| "input": "generation/niches/text-to-sql/evals/spider_dev_qwen35_lora_800/base_predictions_full.jsonl", |
| "no_sql_found": 194, |
| "out": "generation/niches/text-to-sql/evals/spider_dev_qwen35_lora_800/base_predictions_full_sql_extracted.jsonl", |
| "total": 1034, |
| "unchanged": 0 |
| }, |
| "execution_accuracy_delta_points": 17.02, |
| "fine_tuned": { |
| "correct": 598, |
| "execution_accuracy": 0.5783, |
| "total": 1034, |
| "valid_sql": 809, |
| "valid_sql_rate": 0.7824 |
| }, |
| "fine_tuned_extraction": { |
| "changed": 1014, |
| "input": "generation/niches/text-to-sql/evals/spider_dev_qwen35_lora_800/fine_tuned_predictions_full.jsonl", |
| "no_sql_found": 20, |
| "out": "generation/niches/text-to-sql/evals/spider_dev_qwen35_lora_800/fine_tuned_predictions_full_sql_extracted.jsonl", |
| "total": 1034, |
| "unchanged": 0 |
| }, |
| "valid_sql_delta_points": 22.82 |
| } |
| } |
|
|
