Deploy NL_SQL HEAD to HF Space

app/streamlit_app.py

@@ -61,7 +61,7 @@ I18N: dict[str, dict[str, str]] = {
         "metric_percent": "100%",
         "metric_caption": "30 dev + 30 held-out, balanced split, all ten query categories at 100% on the free-tier codestral pipeline.",
         "research_kicker": "BIRD Mini-Dev research benchmark",
-        "research_value": "77.5% / 200",
+        "research_value": "79.0% / 200",
         "research_caption": "Hybrid pipeline: codestral + Sonnet on challenging tier + cross-provider voting + grounded-critique directed retry + Sonnet 4.6 bridge on the remaining fails. +29.2pp over the GPT-4 zero-shot reference (47.8%), $0 external cost.",
         "settings_header": "Settings",
         "db_label": "Database",
@@ -131,7 +131,7 @@ I18N: dict[str, dict[str, str]] = {
         "metric_percent": "100%",
         "metric_caption": "30 dev + 30 held-out, сбалансированный сплит, все десять категорий запросов на 100% через бесплатный codestral.",
         "research_kicker": "Исследовательский бенчмарк BIRD Mini-Dev",
-        "research_value": "77.5% / 200",
+        "research_value": "79.0% / 200",
         "research_caption": "Гибрид: codestral + Sonnet на challenging-тире + кросс-провайдер voting + grounded-critique directed retry + Sonnet 4.6 bridge на оставшихся фейлах. +29.2 п.п. над zero-shot GPT-4 (47.8%), внешние расходы — ноль.",
         "settings_header": "Настройки",
         "db_label": "База данных",

chroma_data/chroma.sqlite3

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:93b193ae7c2c3836ac94b13ad2ade2369ba0fd20377e09623e54504418303520
+oid sha256:d7f72c510d8781191aa4e8173bee8ba4550f99d4f1f5df7562c5191435058aea
 size 18161664

chroma_data/fc9668d3-4384-40d9-aa8d-0010807a5a68/data_level0.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ecc11004a1564969700ee05e3c2d1ab196c5198a9a8f4d5a1bf489b9b7046e57
+oid sha256:9f7a9f240805365c053e0d463ec92d146c94b4dc22ec2f05297745fc44c5dcd2
 size 423600

chroma_data/fc9668d3-4384-40d9-aa8d-0010807a5a68/length.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d9142387334b53a34226a53a60878980670c9a2a28a56cd9f8ddf2e7d44f3975
+oid sha256:1a7518686c00d3009a5c05e60bca24d2903519877de75996d3df0d945f18bc8b
 size 400

docs/03_eval_methodology.md

@@ -96,24 +96,24 @@
 
 ### 4.2 Что репортится для каждой конфигурации
 
-Шаблон с реальными числами для финальной shipped конфигурации (G + multi-vote + critique + selfcon + Sonnet bridge + selective fewshot expansion, n=200, seed=0, отчёт 2026-05-17 v7):
+Шаблон с реальными числами для финальной shipped конфигурации (G + multi-vote + critique + selfcon + Sonnet bridge + selective fewshot expansion + cross-Groq voting, n=200, seed=0, отчёт 2026-05-17 night v8):
 
 ```
-Configuration G_hybrid+multi-vote+critique+selfcon+sonnet+fewshot5  (final shipped path)
-  EA (overall):           77.5%   (155/200, +29.7pp vs GPT-4 zero-shot 47.8%)
-  EA (simple):            89.6%   (60/67)
-  EA (moderate):          74.7%   (74/99)
-  EA (challenging):       61.8%   (21/34)
-  EA (SQLite only):       77.5%   (BIRD Mini-Dev is SQLite-only)
-  Voting rescues:         41/200  (frozen-fail directed retry across vote buckets)
+Configuration G_hybrid+multi-vote+critique+selfcon+sonnet+fewshot5+groq3  (final shipped path)
+  EA (overall):           79.0%   (158/200, +31.2pp vs GPT-4 zero-shot 47.8%)
+  EA (simple):            91.0%   (61/67)
+  EA (moderate):          75.8%   (75/99)
+  EA (challenging):       64.7%   (22/34)
+  EA (SQLite only):       79.0%   (BIRD Mini-Dev is SQLite-only)
+  Voting rescues:         44/200  (frozen-fail directed retry across vote buckets)
   Schema Recall@5:        100.0%
   SQL Validity Rate:      100.0%
-  First-pass / Final EA:  47.0 / 77.5   (codestral A baseline → final)
+  First-pass / Final EA:  47.0 / 79.0   (codestral A baseline → final)
   Latency P50 / P95:      ~65 ms cache-hit / dozens of seconds on Sonnet-rescued tier
   Cost per query:         $0    (Mistral free + Groq free + Perplexity Pro browser bridge)
 ```
 
-Per-bucket lifts that compose the 77.5% headline:
+Per-bucket lifts that compose the 79.0% headline:
 
 ```
 A (codestral full_schema)                         47.0%   baseline
@@ -128,6 +128,7 @@ G + Sonnet challenging tier hybrid                57.0%   +0.5pp
 + Mistral self-consistency                        72.5%   +0.5pp
 + Sonnet rescue on frozen-fail tail               77.0%   +4.5pp (9 rescues, 0 regressions)
 + selective fewshot_top_k=5 on residue            77.5%   +0.5pp (1 rescue / 0 regressions, qid=1500)
++ cross-Groq voting on residue (llama3.3-70b+qwen3) 79.0% +1.5pp (3 rescues / 0 regressions, qids 219+352+366)
 ```
 
 **Selective fewshot expansion note:** глобальный `fewshot_top_k=5` (вместо
@@ -307,11 +308,12 @@ Business hints:
 | G + Sonnet challenging hybrid                    | 57.0%       | 71.6%  | 53.5%    | 38.2%       |
 | + multi-vote + grounded-critique + selfcon       | 72.5%       | 86.6%  | 70.7%    | 55.9%       |
 | + Sonnet rescue on frozen-fail tail              | 77.0%       | 88.1%  | 74.7%    | 61.8%       |
-| **+ selective fewshot_top_k=5 on residue (final)** | **77.5%**   | **89.6%** | **74.7%** | **61.8%** |
+| + selective fewshot_top_k=5 on residue           | 77.5%       | 89.6%  | 74.7%    | 61.8%       |
+| **+ cross-Groq llama3.3-70b + qwen3 voting (final)** | **79.0%**   | **91.0%** | **75.8%** | **64.7%** |
 | Reference: GPT-4 zero-shot (BIRD paper)          | 47.8%       | —      | —        | —           |
 | Reference: paid SOTA CHESS/Distillery 2024       | 73–76%      | —      | —        | —           |
 
-Final shipped configuration matches `eval/reports/2026-05-17/hybrid+multi-vote+critique+selfcon+sonnet+fewshot5-v7.json` — see also memory note `project_nl_sql_quality_push`.
+Final shipped configuration matches `eval/reports/2026-05-17/hybrid-vote-critique-selfcon-sonnet-fewshot5-groq3-v8.json` — see also memory note `project_nl_sql_quality_push`.
 
 Config B (BM25 cards) is intentionally absent from the shipped pipeline — dense retrieval (config C) was strictly superior in pilot runs and BM25 would only widen the prompt with no recall lift. `Configuration.B_BM25` enum and `run_config_b` (NotImplementedError) are kept so the A–E ladder reads as documented, but the production path is A → C → D → G → hybrid → voting/critique/selfcon → Sonnet rescue.
 

docs/NEXT_SESSION.md

@@ -3,87 +3,76 @@
 > Один лист, без воды. Берёшь, делаешь, обновляешь `SESSION_HANDOFF.md`,
 > удаляешь этот файл (или переписываешь под следующий sprint).
 
-## Контекст на 2026-05-17
-
-- HEAD `298614f` (после 3ca3612 + docs:refresh + HF deploy session)
-- BIRD Mini-Dev n=200: **77.5% EA** (155/200), per tier 89.6/74.7/61.8 (v7 = v6 + selective fewshot_top_k=5 on residue, +1 rescue qid=1500)
-- 270 pytest pass (+20 за scalar label classifier + 3 drift guards), ruff + mypy strict clean
-- Streamlit UI переписан в editorial monochrome + EN/RU; scalar metric labels гуманизированы
-- Portfolio screenshots EN/RU в `docs/ui-2026-05-17-{en,ru}.png` привязаны в README hero
-- **Live demo на HF Spaces:** <https://liovina-nl-sql.hf.space> (deploy headless через `.deploy_hf.py`, see § P0)
-- 2026-05-12 audit P1 backlog закрыт (build_index sample-size drift, CI lint scope, pinned requirements, BM25 cleanup в methodology)
-- GraceKelly Sonnet bridge доказан рабочим (9 rescues / 0 regressions)
-
-## ~~P0 — Streamlit Cloud deploy~~ **CLOSED 2026-05-17**
-
-Live: <https://liovina-nl-sql.hf.space> (HF Spaces, Docker runtime, free
-tier). Headless deploy через `huggingface_hub.HfApi`: `.deploy_hf.py`
-создаёт Space `liovina/nl-sql` с `space_sdk=docker`, прокидывает
-`MISTRAL_API_KEY` через `add_space_secret`, заливает 214 MB кода + данных
-с auto-LFS, генерирует HF README frontmatter (`sdk: docker, app_port:
-7860`) + Dockerfile (`python:3.12-slim`, `pip -r requirements.txt`,
-`streamlit run app/streamlit_app.py --server.port 7860`).
-
-Streamlit Cloud не пошёл (требует Gmail OAuth, у Юлии не открывается),
-Fly.io/Railway/Render не пошли (sign-up через email OAuth). HF — у
-Юлии уже залогинен (`liovina`, token в `~/.cache/huggingface/token`),
-поэтому весь deploy ушёл headless без единого клика.
-
-Repush после правок: повторить `uv run python .deploy_hf.py` — `exist_ok`
-+ idempotent upload_folder корректно перезаписывают Space.
-
-## P1 — портфолио-материалы под новый UI
-
-Хороший shot нового UI = sellable артефакт. Конкретно:
-
-1. ~~**Один screenshot EN + один RU** под hero-section какого-нибудь
-   проектного проф-сайта или LinkedIn. 1440×900 viewport, default DB
-   `bird_california_schools`, без открытых expanders. Сохранить под
-   `docs/ui-2026-05-13-{en,ru}.png` и привязать в README.~~ **Закрыто
-   2026-05-17:** `docs/ui-2026-05-17-{en,ru}.png` сняты через Playwright
-   headless Streamlit, привязаны в README hero-секции.
-2. **Короткий AutoReel-ролик** (`D:\AutoReel\`) с тремя shots:
-   (a) headline + metric block,
-   (b) sample-click → answer render,
-   (c) language toggle EN→RU.
-   Memory `feedback_real_product_over_mockup` говорит: реальная запись
-   экрана > HTML-template для проектов с live demo. Если P0 закрыт и
-   live URL есть — записывай live URL, не localhost.
-
-## P2 — quality push past 77% (если есть желание)
-
-Остаток 46 фейлов: 22 row_count_off + 14 filter_or_value + 6 order_by_off
-+ 4 errors. Все «потолочные» — codestral + Sonnet согласуются на
-неверном результате. Реальные рычаги:
-
-| Эксперимент | Ожидание | Стоимость |
-|---|---|---|
-| **GraceKelly: GPT-5.4 на остатке через Perplexity bridge** | +1-3pp; ортогональный к Sonnet, может закрыть другие фейлы | $0 wall, ~50 мин |
-| **BIRD train fewshot expansion** (top_k=5 на failures with `enable_grounded_critique`) | +0-2pp; раньше top_k=5 давал -1pp при глобальном применении, но selective может сыграть | $0 wall, 5 мин |
-| **Question rephrasing through Sonnet → re-feed pipeline** | +0-3pp; BIRD-style формализация вопроса, потом codestral пытается ещё раз | $0 wall, ~50 мин |
-| **Hard fail: row_count_off через explicit JOIN-path hint** | +5-10pp ceiling lift, но требует custom schema-linker (research-grade work, не sprint) | дни-недели |
+## Контекст на 2026-05-17 EOS
+
+- HEAD `e0ea5ad` (после 9370070 + P2.B fewshot5-residue lift + HF Dockerfile fix)
+- BIRD Mini-Dev n=200: **79.0% EA** (158/200), per tier 91.0/75.8/64.7 (v8 = v7 + cross-Groq llama-3.3-70b + qwen3-32b voting на residue, +3 rescues qids 219+352+366)
+- **Live demo:** <https://liovina-nl-sql.hf.space> RUNNING, headline 79.0% / 200
+- 270 pytest pass, ruff + mypy strict clean (55 source files)
+- Streamlit UI editorial monochrome + EN/RU (закрыто 2026-05-13)
+- Portfolio screenshots: `docs/ui-2026-05-17-{en,ru}.png` (local Streamlit) + `docs/ui-live-en.png` (live HF)
+- P2.B + P0 deploy closed автономно 2026-05-17
+
+## P1 — оставшийся портфолио-материал
+
+1. ~~Screenshots EN+RU local Streamlit~~ ✓ закрыто 2026-05-17.
+2. **Короткий live-URL ролик** (`D:\AutoReel\` шаблон ИЛИ Playwright video record):
+   - shot A: hero (headline 77.5% + metric block)
+   - shot B: sample-click → SQL + answer render
+   - shot C: EN→RU toggle
+   - **Источник: live URL** (`https://liovina-nl-sql.hf.space`), не localhost — memory `feedback_real_product_over_mockup`.
 
-**Не пытаться повторять:**
+## P2/P3 — quality push past 79.0% ($0 budget)
+
+Остаток **42 фейла** (после v8): 22 row_count_off + 13 filter_or_value + 5 order_by_off + 2 errors.
+
+| Эксперимент | Статус | Ожидание |
+|---|---|---|
+| Selective `fewshot_top_k=5` on residue | **✓ done v7** (+0.5pp, qid=1500 simple) | — |
+| Cross-Groq voting (llama3.3-70b + qwen3) | **✓ done v8** (+1.5pp, qids 219+352+366) | — |
+| Mistral-large voting on residue | **✗ negative** (TPD/TPM limits — 18 attempted, all same; structural failures unanimous across Mistral models) | — |
+| Wide-schema retry on row_count_off | **✗ negative** (0/20 rescues с critique tоо) | — |
+| codestral fewshot_top_k=7 | **✗ negative** (0/45, top_k=5 насыщает) | — |
+| gpt-oss-120b throttled voting | **✗ TPM limits** (0/24 rescues, prompts 8.5K > 8K TPM gives 413; первая попытка с fewshot=5 ранее дала +1 rescue qid=571, но воспроизвести не вышло — fewshot=3 теряет critical context для cases) | — |
+| GraceKelly: GPT-5.4 via Perplexity bridge | **OPEN** (P3.D) | +1-3pp ортогональный к Sonnet. **Гейт:** Chrome profile свободен. |
+| Question rephrasing through Sonnet → re-feed | **OPEN** (P3.E) | +0-3pp. **Гейт:** GraceKelly bridge live. |
+| row_count_off через explicit JOIN-path hint (custom schema-linker) | **research-grade** (P3.F) | +5-10pp ceiling lift, дни-недели работы. |
+| llama-3.3-70b TPD reset retry | **OPEN** | TPD resets ~24h. На v8-residue ещё 30 unattempted; ожидаемо +0-2pp. |
+
+**Не повторять:**
 - Anthropic API direct — out of $0 budget.
-- Wide-schema retry — уже подтверждено saturated.
+- Wide-schema retry — saturated (повторно подтверждено в 2026-05-17 night).
 - Column-count critique — empirically бесполезен (0/19 mismatch).
 - Same-model self-consistency — plateau.
+- Mistral-large voting — 2026-05-17 EOS+night зафиксирован negative.
+- codestral fewshot_top_k=7 — 0/45 на v7-residue, top_k=5 саtuрated.
+- gpt-oss-120b voting — TPM 8K жёстко режет, prompts с critique=10K+; нужны без-critique runs (но они теряют lift signal).
 
-## Закрытые тейлы для следующей сессии
+## Что НЕ делать
 
-- `audit_codex_12_05_26.md` ещё не закрыт по P1 пунктам:
-  - sample-size `build_index.py` vs runtime mismatch (открыт)
-  - CI lint app/scripts (открыт)
-  - wide dependency ranges в `requirements.txt` (открыт)
-  Все три — P1 medium, не блокеры; брать вместе с P0 deploy если будет
-  CI-time.
+- Не редизайнить UI. Зафиксирован 2026-05-13.
+- Не коммитить `chroma_data/` byte-level drift от смок-запусков.
+- Не запускать GraceKelly `dry-run → hybrid` без подтверждения, что Chrome-профиль свободен (memory `feedback_user_chrome_assumption`).
+- Не пробовать Mistral-large снова на residue без throttling — free tier даёт ≤2 req/sec, скрипту нужен `--sleep-between` arg.
 
-## Что НЕ делать
+## Quick start если хочется быстрого win
+
+```bash
+# Repush HF Space после правок (idempotent, ~90s до RUNNING):
+uv run python .deploy_hf.py
+
+# Gate:
+uv run pytest -q && uv run ruff check src tests scripts app && uv run mypy --strict src
+
+# Local Streamlit (cache-warm UI):
+make ui
+```
+
+## Deploy quick reference
 
-- Не редизайнить UI повторно. Текущий редизайн принят и зафиксирован.
-- Не коммитить `chroma_data/` byte-level изменения от смок-запусков
-  (они в working tree после каждого Streamlit-run, оставляй
-  uncommitted — реальные перестроения индекса делаются через
-  `scripts/build_index.py` и тогда commit'ятся осознанно).
-- Не запускать GraceKelly `dry-run -> hybrid` бе�� подтверждения, что
-  Chrome-профиль свободен (memory `feedback_user_chrome_assumption`).
+- Live URL: <https://liovina-nl-sql.hf.space>
+- Dashboard: <https://huggingface.co/spaces/liovina/nl-sql>
+- Deploy script: `.deploy_hf.py` (gitignored)
+- HF Dockerfile template: `.tmp/hf_Dockerfile` (важно: `ENV PYTHONPATH=/app/src` для src layout)
+- Mistral key: `D:/TXT/Mistral_API.txt`
+- Полный runbook: `docs/SESSION_HANDOFF.md § Deploy — DONE`

docs/SESSION_HANDOFF.md

@@ -1,4 +1,19 @@
-# NL_SQL — Session Handoff (2026-05-13, multi-vote + grounded-critique + Sonnet bridge + UI redesign → 77.0% BIRD)
+# NL_SQL — Session Handoff (2026-05-17 EOS: 77.5% BIRD + live HF Space + autonomous deploy/lift sprint)
+
+> **Tl;dr 2026-05-17 EOS:** P0 closed (live demo on HF Spaces, headless API
+> deploy), P2.B closed (+1 selective fewshot rescue → 77.5% n=200). Full
+> gate green: 270 pytest, ruff + mypy strict clean. Mistral-large voting
+> on residue tried → negative (rate-limit + structural agreement with
+> codestral). Open: P2.A (GraceKelly GPT-5.4) and P2.C (Sonnet
+> rephrasing) gated on Chrome profile confirmation; P2.D (custom
+> schema-linker for row_count_off) research-grade.
+>
+> Read `docs/NEXT_SESSION.md` for the action list and historic context
+> in this file below.
+
+---
+
+# Historic handoff: 2026-05-13 (multi-vote + grounded-critique + Sonnet bridge + UI redesign → 77.0% BIRD)
 
 > Read this first when picking up. It's the single source of truth for
 > "where we stopped" and "what to do next". When you take action, update

eval/reports/2026-05-17/groq-gptoss120b-throttled-on-v7.json

@@ -0,0 +1,370 @@
+{
+  "alt_model": "groq:openai/gpt-oss-120b+grounded_critique+fewshot3",
+  "summary": {
+    "voted_better": 0,
+    "voted_worse": 0,
+    "voted_same": 24
+  },
+  "records": [
+    {
+      "question_id": 25,
+      "db_id": "california_schools",
+      "difficulty": "moderate",
+      "question": "Name schools in Riverside which the average of average math score for SAT is grater than 400, what is the funding type of these schools?",
+      "gold_sql": "SELECT T1.sname, T2.`Charter Funding Type` FROM satscores AS T1 INNER JOIN frpm AS T2 ON T1.cds = T2.CDSCode WHERE T2.`District Name` LIKE 'Riverside%' GROUP BY T1.sname, T2.`Charter Funding Type` HAVING CAST(SUM(T1.AvgScrMath) AS REAL) / COUNT(T1.cds) > 400",
+      "baseline_pred": "SELECT s.FundingType FROM schools AS s JOIN satscores AS t ON s.CDSCode = t.cds WHERE s.City = 'Riverside' AND t.AvgScrMath > 400 GROUP BY s.FundingType HAVING AVG(t.AvgScrMath) > 400",
+      "alt_pred": "SELECT s.School, s.FundingType FROM schools s JOIN satscores ss ON s.CDSCode = ss.cds WHERE s.County = 'Riverside' AND ss.AvgScrMath > 400",
+      "alt_confidence": 0.93,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 37344.45269999924
+    },
+    {
+      "question_id": 37,
+      "db_id": "california_schools",
+      "difficulty": "moderate",
+      "question": "What is the complete address of the school with the lowest excellence rate? Indicate the Street, City, Zip and State.",
+      "gold_sql": "SELECT T2.Street, T2.City, T2.State, T2.Zip FROM satscores AS T1 INNER JOIN schools AS T2 ON T1.cds = T2.CDSCode ORDER BY CAST(T1.NumGE1500 AS REAL) / T1.NumTstTakr ASC LIMIT 1",
+      "baseline_pred": "SELECT Street, City, Zip, State FROM schools WHERE CDSCode = (SELECT cds FROM satscores WHERE NumTstTakr > 0 ORDER BY (NumGE1500 * 1.0 / NumTstTakr) ASC LIMIT 1)",
+      "alt_pred": "SELECT s.\"Street\", s.\"City\", s.\"Zip\", s.\"State\" FROM satscores AS ss JOIN schools AS s ON ss.cds = s.CDSCode ORDER BY (ss.NumGE1500*1.0)/NULLIF(ss.NumTstTakr,0) ASC LIMIT 1",
+      "alt_confidence": 0.93,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 38318.087799998466
+    },
+    {
+      "question_id": 77,
+      "db_id": "california_schools",
+      "difficulty": "moderate",
+      "question": "Which schools served a grade span of Kindergarten to 9th grade in the county of Los Angeles and what is its Percent (%) Eligible FRPM (Ages 5-17)?",
+      "gold_sql": "SELECT T2.School, T1.`FRPM Count (Ages 5-17)` * 100 / T1.`Enrollment (Ages 5-17)` FROM frpm AS T1 INNER JOIN schools AS T2 ON T1.CDSCode = T2.CDSCode WHERE T2.County = 'Los Angeles' AND T2.GSserved = 'K-9'",
+      "baseline_pred": "SELECT f.`School Name`, (f.`FRPM Count (Ages 5-17)` / f.`Enrollment (Ages 5-17)`) * 100 AS `Percent (%) Eligible FRPM (Ages 5-17)` FROM frpm f JOIN schools s ON f.CDSCode = s.CDSCode WHERE f.`County Name` = 'Los Angeles' AND f.`Low Grade` = 'K' AND f.`High Grade` = '9'",
+      "alt_pred": "SELECT \"School Name\", CASE WHEN \"Enrollment (Ages 5-17)\" > 0 THEN (\"FRPM Count (Ages 5-17)\" / \"Enrollment (Ages 5-17)\" ) * 100 END AS \"Percent (%) Eligible FRPM (Ages 5-17)\" FROM frpm WHERE \"County Name\" = 'Los Angeles' AND \"Low Grade\" = 'K' AND \"High Grade\" LIKE '9%'",
+      "alt_confidence": 0.86,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 36778.51229999942
+    },
+    {
+      "question_id": 173,
+      "db_id": "financial",
+      "difficulty": "challenging",
+      "question": "How often does account number 3 request an account statement to be released? What was the aim of debiting 3539 in total?",
+      "gold_sql": "SELECT T1.frequency, T2.k_symbol FROM account AS T1 INNER JOIN (SELECT account_id, k_symbol, SUM(amount) AS total_amount FROM `order` GROUP BY account_id, k_symbol) AS T2 ON T1.account_id = T2.account_id WHERE T1.account_id = 3 AND T2.total_amount = 3539",
+      "baseline_pred": "SELECT COUNT(*) AS frequency, k_symbol AS purpose FROM trans WHERE account_id = 3 AND type = 'VYDAJ' AND k_symbol = 'SLUZBY' GROUP BY k_symbol",
+      "alt_pred": "SELECT (SELECT COUNT(*) FROM \"trans\" WHERE \"account_id\" = 3 AND \"k_symbol\" LIKE '%statement%') AS request_count, (SELECT GROUP_CONCAT(DISTINCT \"k_symbol\") FROM \"trans\" WHERE \"amount\" = 3539 AND \"type\" = 'VYBER') AS purpose",
+      "alt_confidence": 0.52,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 36308.96419999954
+    },
+    {
+      "question_id": 207,
+      "db_id": "toxicology",
+      "difficulty": "challenging",
+      "question": "What elements are in a double type bond?",
+      "gold_sql": "SELECT DISTINCT T1.element FROM atom AS T1 INNER JOIN bond AS T2 ON T1.molecule_id = T2.molecule_id INNER JOIN connected AS T3 ON T1.atom_id = T3.atom_id WHERE T2.bond_type = '='",
+      "baseline_pred": "SELECT DISTINCT a.element FROM atom a JOIN bond b ON a.molecule_id = b.molecule_id JOIN connected c ON b.bond_id = c.bond_id WHERE b.bond_type = '='",
+      "alt_pred": "SELECT DISTINCT a.element FROM atom AS a JOIN connected AS c ON a.atom_id = c.atom_id OR a.atom_id = c.atom_id2 JOIN bond AS b ON c.bond_id = b.bond_id WHERE b.bond_type = '='",
+      "alt_confidence": 0.99,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 13758.136599999489
+    },
+    {
+      "question_id": 219,
+      "db_id": "toxicology",
+      "difficulty": "challenging",
+      "question": "What is the percentage of carcinogenic molecules in triple type bonds?",
+      "gold_sql": "SELECT CAST(COUNT(DISTINCT CASE WHEN T2.label = '+' THEN T2.molecule_id ELSE NULL END) AS REAL) * 100 / COUNT(DISTINCT T2.molecule_id) FROM atom AS T1 INNER JOIN molecule AS T2 ON T1.molecule_id = T2.molecule_id INNER JOIN bond AS T3 ON T2.molecule_id = T3.molecule_id WHERE T3.bond_type = '#'",
+      "baseline_pred": "SELECT CAST(SUM(CASE WHEN b.bond_type = '#' THEN 1 ELSE 0 END) AS REAL) * 100.0 / COUNT(b.bond_id) FROM bond b INNER JOIN atom a ON b.molecule_id = a.molecule_id INNER JOIN molecule m ON a.molecule_id = m.molecule_id WHERE m.label = '+'",
+      "alt_pred": "SELECT (CAST(SUM(CASE WHEN \"bond\".\"bond_type\" = '#' THEN 1 ELSE 0 END) AS REAL) * 100.0) / COUNT(\"bond\".\"bond_id\") AS percent FROM \"bond\" JOIN \"molecule\" ON \"bond\".\"molecule_id\" = \"molecule\".\"molecule_id\" WHERE \"molecule\".\"label\" = '+'",
+      "alt_confidence": 0.98,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 5612.1505000010075
+    },
+    {
+      "question_id": 571,
+      "db_id": "codebase_community",
+      "difficulty": "moderate",
+      "question": "For the user No.24, how many times is the number of his/her posts compared to his/her votes?",
+      "gold_sql": "SELECT CAST(COUNT(DISTINCT T2.Id) AS REAL) / COUNT(DISTINCT T1.Id) FROM votes AS T1 INNER JOIN posts AS T2 ON T1.UserId = T2.OwnerUserId WHERE T1.UserId = 24",
+      "baseline_pred": "SELECT CAST(COUNT(DISTINCT p.Id) AS REAL) / NULLIF(COUNT(DISTINCT v.Id), 0) AS post_to_vote_ratio FROM posts p LEFT JOIN votes v ON p.Id = v.PostId WHERE p.OwnerUserId = 24",
+      "alt_pred": "SELECT (COUNT(DISTINCT \"posts\".\"Id\") * 1.0) / NULLIF(COUNT(\"votes\".\"Id\"),0) AS ratio FROM \"posts\" LEFT JOIN \"votes\" ON \"posts\".\"Id\" = \"votes\".\"PostId\" WHERE \"posts\".\"OwnerUserId\" = 24",
+      "alt_confidence": 0.99,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 2342.348399999537
+    },
+    {
+      "question_id": 584,
+      "db_id": "codebase_community",
+      "difficulty": "moderate",
+      "question": "Write all the comments left by users who edited the post titled 'Why square the difference instead of taking the absolute value in standard deviation?'",
+      "gold_sql": "SELECT T2.Comment FROM posts AS T1 INNER JOIN postHistory AS T2 ON T1.Id = T2.PostId WHERE T1.Title = 'Why square the difference instead of taking the absolute value in standard deviation?'",
+      "baseline_pred": "SELECT comments.Text FROM comments JOIN posts ON comments.PostId = posts.Id WHERE posts.Title = 'Why square the difference instead of taking the absolute value in standard deviation?'",
+      "alt_pred": "SELECT \"Text\" FROM \"comments\" WHERE \"UserId\" IN (SELECT DISTINCT \"UserId\" FROM \"postHistory\" WHERE \"PostId\" = (SELECT \"Id\" FROM \"posts\" WHERE \"Title\" = 'Why square the difference instead of taking the absolute value in standard deviation?') AND \"UserId\" IS NOT NULL)",
+      "alt_confidence": 0.9,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 28731.873999999152
+    },
+    {
+      "question_id": 595,
+      "db_id": "codebase_community",
+      "difficulty": "moderate",
+      "question": "Which user have only one post history per post and having at least 1000 views?",
+      "gold_sql": "SELECT T2.UserId FROM users AS T1 INNER JOIN postHistory AS T2 ON T1.Id = T2.UserId INNER JOIN posts AS T3 ON T2.PostId = T3.Id WHERE T3.ViewCount >= 1000 GROUP BY T2.UserId HAVING COUNT(DISTINCT T2.PostHistoryTypeId) = 1",
+      "baseline_pred": "SELECT DISTINCT ph.UserId FROM postHistory ph JOIN posts p ON ph.PostId = p.Id WHERE p.ViewCount >= 1000 AND ph.UserId IS NOT NULL GROUP BY ph.UserId, ph.PostId HAVING COUNT(*) = 1",
+      "alt_pred": "SELECT DISTINCT u.UserId FROM (SELECT UserId FROM postHistory GROUP BY UserId HAVING COUNT(*) = COUNT(DISTINCT PostId)) u JOIN postHistory ph ON ph.UserId = u.UserId JOIN posts p ON ph.PostId = p.Id WHERE p.ViewCount >= 1000",
+      "alt_confidence": 0.78,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 42098.9497999999
+    },
+    {
+      "question_id": 672,
+      "db_id": "codebase_community",
+      "difficulty": "moderate",
+      "question": "Among the users located in United Kingdom, how many users whose post have a total favorite amount of 4 or more?",
+      "gold_sql": "SELECT COUNT(T1.Id) FROM users AS T1 INNER JOIN posts AS T2 ON T1.Id = T2.OwnerUserId WHERE T1.Location = 'United Kingdom' AND T2.FavoriteCount >= 4",
+      "baseline_pred": "SELECT COUNT(DISTINCT u.Id) FROM users u JOIN posts p ON u.Id = p.OwnerUserId WHERE u.Location = 'United Kingdom' AND p.FavoriteCount >= 4",
+      "alt_pred": "SELECT COUNT(*) FROM (SELECT u.Id FROM users u JOIN posts p ON p.OwnerUserId = u.Id WHERE u.Location = 'United Kingdom' GROUP BY u.Id HAVING SUM(COALESCE(p.FavoriteCount,0)) >= 4) sub",
+      "alt_confidence": 0.78,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 29584.98480000162
+    },
+    {
+      "question_id": 694,
+      "db_id": "codebase_community",
+      "difficulty": "moderate",
+      "question": "Provide the text of the latest 10 comments to the post with the title 'Analysing wind data with R' and the display name of the user who left it.",
+      "gold_sql": "SELECT T3.Text, T1.DisplayName FROM users AS T1 INNER JOIN posts AS T2 ON T1.Id = T2.OwnerUserId INNER JOIN comments AS T3 ON T2.Id = T3.PostId WHERE T2.Title = 'Analysing wind data with R' ORDER BY T1.CreationDate DESC LIMIT 10",
+      "baseline_pred": "SELECT comments.Text, comments.UserDisplayName FROM comments JOIN posts ON comments.PostId = posts.Id WHERE posts.Title = 'Analysing wind data with R' ORDER BY comments.CreationDate DESC LIMIT 10",
+      "alt_pred": "SELECT c.Text, u.DisplayName FROM comments AS c JOIN posts AS p ON c.PostId = p.Id LEFT JOIN users AS u ON c.UserId = u.Id WHERE p.Title = 'Analysing wind data with R' ORDER BY c.CreationDate DESC LIMIT 10",
+      "alt_confidence": 0.97,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 35489.011700001356
+    },
+    {
+      "question_id": 743,
+      "db_id": "superhero",
+      "difficulty": "challenging",
+      "question": "What is the percentage of superheroes who act in their own self-interest or make decisions based on their own moral code? Indicate how many of the said superheroes were published by Marvel Comics.",
+      "gold_sql": "SELECT (CAST(COUNT(*) AS REAL) * 100 / (SELECT COUNT(*) FROM superhero)), CAST(SUM(CASE WHEN T2.publisher_name = 'Marvel Comics' THEN 1 ELSE 0 END) AS REAL) FROM superhero AS T1 INNER JOIN publisher AS T2 ON T1.publisher_id = T2.id INNER JOIN alignment AS T3 ON T3.id = T1.alignment_id WHERE T3.alignment = 'Bad'",
+      "baseline_pred": "SELECT CAST(SUM(CASE WHEN a.alignment = 'Bad' THEN 1 ELSE 0 END) AS REAL) * 100 / COUNT(s.id) AS percentage, SUM(CASE WHEN a.alignment = 'Bad' AND p.publisher_name = 'Marvel Comics' THEN 1 ELSE 0 END) FROM superhero s INNER JOIN alignment a ON s.alignment_id = a.id LEFT JOIN publisher p ON s.publisher_id = p.id",
+      "alt_pred": "SELECT CAST(SUM(CASE WHEN a.alignment = 'Bad' THEN 1 ELSE 0 END) AS REAL) * 100.0 / COUNT(*) AS percentage, SUM(CASE WHEN a.alignment = 'Bad' AND p.publisher_name = 'Marvel Comics' THEN 1 ELSE 0 END) AS marvel_bad_count FROM superhero s LEFT JOIN alignment a ON s.alignment_id = a.id LEFT JOIN publisher p ON s.publisher_id = p.id",
+      "alt_confidence": 0.98,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 21804.846799999723
+    },
+    {
+      "question_id": 866,
+      "db_id": "formula_1",
+      "difficulty": "moderate",
+      "question": "Who was the player that got the lap time of 0:01:27 in the race No. 161? Show his introduction website.",
+      "gold_sql": "SELECT DISTINCT T2.forename, T2.surname, T2.url FROM lapTimes AS T1 INNER JOIN drivers AS T2 ON T2.driverId = T1.driverId WHERE T1.raceId = 161 AND T1.time LIKE '1:27%'",
+      "baseline_pred": "SELECT drivers.url FROM lapTimes JOIN drivers ON lapTimes.driverId = drivers.driverId WHERE lapTimes.raceId = 161 AND lapTimes.time LIKE '1:27%'",
+      "alt_pred": "SELECT DISTINCT drivers.url FROM lapTimes JOIN drivers ON lapTimes.driverId = drivers.driverId WHERE lapTimes.raceId = 161 AND lapTimes.time LIKE '1:27%'",
+      "alt_confidence": 0.95,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 31899.09280000029
+    },
+    {
+      "question_id": 894,
+      "db_id": "formula_1",
+      "difficulty": "moderate",
+      "question": "What is the best lap time recorded? List the driver and race with such recorded lap time.",
+      "gold_sql": "SELECT T2.milliseconds, T1.forename, T1.surname, T3.name FROM drivers AS T1 INNER JOIN lapTimes AS T2 ON T1.driverId = T2.driverId INNER JOIN races AS T3 ON T2.raceId = T3.raceId ORDER BY T2.milliseconds ASC LIMIT 1",
+      "baseline_pred": "SELECT drivers.forename, drivers.surname, races.name FROM lapTimes JOIN drivers ON lapTimes.driverId = drivers.driverId JOIN races ON lapTimes.raceId = races.raceId WHERE lapTimes.milliseconds = (SELECT MIN(milliseconds) FROM lapTimes) LIMIT 1",
+      "alt_pred": "SELECT drivers.forename, drivers.surname, races.name FROM lapTimes JOIN drivers ON lapTimes.driverId = drivers.driverId JOIN races ON lapTimes.raceId = races.raceId WHERE lapTimes.milliseconds = (SELECT MIN(milliseconds) FROM lapTimes)",
+      "alt_confidence": 0.99,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 40876.36140000177
+    },
+    {
+      "question_id": 896,
+      "db_id": "formula_1",
+      "difficulty": "challenging",
+      "question": "Calculate the percentage whereby Hamilton was not at the 1st track of the the f1 circuit since 2010.",
+      "gold_sql": "SELECT CAST(COUNT(CASE WHEN T2.position <> 1 THEN T2.position END) AS REAL) * 100 / COUNT(T2.driverStandingsId) FROM races AS T1 INNER JOIN driverStandings AS T2 ON T2.raceId = T1.raceId INNER JOIN drivers AS T3 ON T3.driverId = T2.driverId WHERE T3.surname = 'Hamilton' AND T1.year >= 2010",
+      "baseline_pred": "SELECT CAST(SUM(CASE WHEN r.positionOrder > 1 THEN 1 ELSE 0 END) AS REAL) * 100.0 / COUNT(*) FROM results r INNER JOIN drivers d ON r.driverId = d.driverId INNER JOIN races ra ON r.raceId = ra.raceId WHERE d.surname = 'Hamilton' AND ra.year >= 2010",
+      "alt_pred": "SELECT CAST(SUM(CASE WHEN \"results\".\"position\" > 1 THEN 1 ELSE 0 END) AS REAL) * 100.0 / COUNT(*) AS percentage FROM \"results\" JOIN \"drivers\" ON \"results\".\"driverId\" = \"drivers\".\"driverId\" JOIN \"races\" ON \"results\".\"raceId\" = \"races\".\"raceId\" WHERE \"drivers\".\"surname\" = 'Hamilton' AND \"races\".\"year\" >= 2010",
+      "alt_confidence": 0.99,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 40322.427699999025
+    },
+    {
+      "question_id": 902,
+      "db_id": "formula_1",
+      "difficulty": "simple",
+      "question": "Which race was Alex Yoong in when he was in track number less than 20?",
+      "gold_sql": "SELECT T1.name FROM races AS T1 INNER JOIN driverStandings AS T2 ON T2.raceId = T1.raceId INNER JOIN drivers AS T3 ON T3.driverId = T2.driverId WHERE T3.forename = 'Alex' AND T3.surname = 'Yoong' AND T2.position < 20",
+      "baseline_pred": "SELECT races.name FROM races JOIN results ON races.raceId = results.raceId JOIN drivers ON results.driverId = drivers.driverId WHERE drivers.forename = 'Alex' AND drivers.surname = 'Yoong' AND results.position < 20",
+      "alt_pred": "SELECT races.name FROM results JOIN drivers ON results.driverId = drivers.driverId JOIN races ON results.raceId = races.raceId WHERE drivers.forename = 'Alex' AND drivers.surname = 'Yoong' AND results.position < 20",
+      "alt_confidence": 0.97,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 39025.16929999911
+    },
+    {
+      "question_id": 930,
+      "db_id": "formula_1",
+      "difficulty": "simple",
+      "question": "In which Formula_1 race did Lewis Hamilton rank the highest?",
+      "gold_sql": "SELECT name FROM races WHERE raceId IN ( SELECT raceId FROM results WHERE rank = 1 AND driverId = ( SELECT driverId FROM drivers WHERE forename = 'Lewis' AND surname = 'Hamilton' ) )",
+      "baseline_pred": "SELECT races.name FROM races JOIN results ON races.raceId = results.raceId JOIN drivers ON results.driverId = drivers.driverId WHERE drivers.forename = 'Lewis' AND drivers.surname = 'Hamilton' ORDER BY results.rank ASC LIMIT 1",
+      "alt_pred": "SELECT races.name FROM results JOIN drivers ON results.driverId = drivers.driverId JOIN races ON results.raceId = races.raceId WHERE drivers.forename = 'Lewis' AND drivers.surname = 'Hamilton' AND results.rank IS NOT NULL ORDER BY results.rank ASC LIMIT 1",
+      "alt_confidence": 0.95,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 36837.869500001034
+    },
+    {
+      "question_id": 988,
+      "db_id": "formula_1",
+      "difficulty": "challenging",
+      "question": "List down top 3 German drivers who has the shortest average pit stop duration and were born between 1980-1985.",
+      "gold_sql": "SELECT T2.forename, T2.surname FROM pitStops AS T1 INNER JOIN drivers AS T2 on T1.driverId = T2.driverId WHERE T2.nationality = 'German' AND STRFTIME('%Y', T2.dob) BETWEEN '1980' AND '1985' GROUP BY T2.forename, T2.surname ORDER BY AVG(T1.duration) LIMIT 3",
+      "baseline_pred": "SELECT d.forename || ' ' || d.surname AS full_name FROM pitStops p JOIN drivers d ON p.driverId = d.driverId WHERE strftime('%Y', d.dob) BETWEEN '1980' AND '1985' AND d.nationality = 'German' GROUP BY d.driverId, d.forename, d.surname ORDER BY AVG(p.milliseconds) ASC LIMIT 3",
+      "alt_pred": "SELECT d.forename || ' ' || d.surname AS full_name, AVG(p.milliseconds) AS avg_duration FROM drivers d JOIN pitStops p ON p.driverId = d.driverId WHERE d.nationality = 'German' AND CAST(strftime('%Y', d.dob) AS INTEGER) BETWEEN 1980 AND 1985 GROUP BY d.driverId ORDER BY avg_duration ASC LIMIT 3",
+      "alt_confidence": 0.97,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 73423.88260000007
+    },
+    {
+      "question_id": 990,
+      "db_id": "formula_1",
+      "difficulty": "challenging",
+      "question": "What is the constructor reference name of the champion in the 2009 Singapore Grand Prix? Please give its website.",
+      "gold_sql": "SELECT T3.constructorRef, T3.url FROM results AS T1 INNER JOIN races AS T2 on T1.raceId = T2.raceId INNER JOIN constructors AS T3 on T1.constructorId = T3.constructorId WHERE T2.name = 'Singapore Grand Prix' AND T2.year = 2009 AND T1.time LIKE '_:%:__.___'",
+      "baseline_pred": "SELECT constructors.constructorRef, constructors.url FROM results INNER JOIN races ON results.raceId = races.raceId INNER JOIN constructors ON results.constructorId = constructors.constructorId WHERE races.year = 2009 AND races.name = 'Singapore Grand Prix' AND results.positionOrder = 1 ORDER BY results.time DESC LIMIT 1",
+      "alt_pred": "SELECT c.constructorRef, c.url FROM results r JOIN races ra ON r.raceId = ra.raceId JOIN constructors c ON r.constructorId = c.constructorId WHERE ra.year = 2009 AND ra.name = 'Singapore Grand Prix' AND r.positionOrder = 1",
+      "alt_confidence": 0.97,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 40548.26719999983
+    },
+    {
+      "question_id": 1168,
+      "db_id": "thrombosis_prediction",
+      "difficulty": "challenging",
+      "question": "The oldest SJS patient's medical laboratory work was completed on what date, and what age was the patient when they initially arrived at the hospital?",
+      "gold_sql": "SELECT T1.Date, STRFTIME('%Y', T2.`First Date`) - STRFTIME('%Y', T2.Birthday),T2.Birthday FROM Laboratory AS T1 INNER JOIN Patient AS T2 ON T1.ID = T2.ID WHERE T2.Diagnosis = 'SJS' AND T2.Birthday IS NOT NULL ORDER BY T2.Birthday ASC LIMIT 1",
+      "baseline_pred": "SELECT L.Date, (strftime('%Y', P.\"First Date\") - strftime('%Y', P.Birthday)) FROM Patient P INNER JOIN Laboratory L ON P.ID = L.ID WHERE P.Diagnosis = 'SJS' AND P.Birthday = (SELECT MIN(Birthday) FROM Patient WHERE Diagnosis = 'SJS') ORDER BY L.Date ASC LIMIT 1",
+      "alt_pred": "SELECT l.Date, (strftime('%Y', p.\"First Date\") - strftime('%Y', p.Birthday)) AS age_at_arrival FROM Patient p JOIN Laboratory l ON p.ID = l.ID WHERE p.Diagnosis = 'SJS' AND p.Birthday = (SELECT MIN(p2.Birthday) FROM Patient p2 WHERE p2.Diagnosis = 'SJS') ORDER BY l.Date DESC LIMIT 1",
+      "alt_confidence": 0.78,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 30448.83469999877
+    },
+    {
+      "question_id": 1205,
+      "db_id": "thrombosis_prediction",
+      "difficulty": "moderate",
+      "question": "Was the patient with the number 57266's uric acid within a normal range?",
+      "gold_sql": "SELECT CASE WHEN (T1.SEX = 'F' AND T2.UA > 6.5) OR (T1.SEX = 'M' AND T2.UA > 8.0) THEN true ELSE false END FROM Patient AS T1 INNER JOIN Laboratory AS T2 ON T1.ID = T2.ID WHERE T1.ID = 57266",
+      "baseline_pred": "SELECT CASE WHEN (T1.UA > 8.0 AND T2.SEX = 'M') OR (T1.UA > 6.5 AND T2.SEX = 'F') THEN 'Yes' ELSE 'No' END AS is_normal FROM Laboratory AS T1 INNER JOIN Patient AS T2 ON T1.ID = T2.ID WHERE T1.ID = 57266 LIMIT 1",
+      "alt_pred": "SELECT CASE WHEN (L.UA > 8.0 AND P.SEX = 'M') OR (L.UA > 6.5 AND P.SEX = 'F') THEN 1 ELSE 0 END AS within_normal_range FROM Laboratory AS L JOIN Patient AS P ON L.ID = P.ID WHERE L.ID = 57266 ORDER BY L.Date DESC LIMIT 1",
+      "alt_confidence": 0.97,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 28915.319499999896
+    },
+    {
+      "question_id": 1235,
+      "db_id": "thrombosis_prediction",
+      "difficulty": "moderate",
+      "question": "What are the patient's diagnosis for those who has lower red blood blood cell? State their ID and age.",
+      "gold_sql": "SELECT DISTINCT T1.Diagnosis, T1.ID , STRFTIME('%Y', CURRENT_TIMESTAMP) -STRFTIME('%Y', T1.Birthday) FROM Patient AS T1 INNER JOIN Laboratory AS T2 ON T1.ID = T2.ID WHERE T2.RBC < 3.5",
+      "baseline_pred": "SELECT P.ID, (strftime('%Y', 'now') - strftime('%Y', P.Birthday)) AS age, P.Diagnosis FROM Patient AS P JOIN Laboratory AS L ON P.ID = L.ID WHERE L.RBC < 3.5",
+      "alt_pred": "SELECT DISTINCT Patient.ID, (strftime('%Y','now') - strftime('%Y', Patient.Birthday)) AS age, Patient.Diagnosis FROM Patient JOIN Laboratory ON Patient.ID = Laboratory.ID WHERE Laboratory.RBC < 3.5",
+      "alt_confidence": 0.97,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 4114.075999999841
+    },
+    {
+      "question_id": 1247,
+      "db_id": "thrombosis_prediction",
+      "difficulty": "challenging",
+      "question": "Among the male patients who have a normal level of white blood cells, how many of them have an abnormal fibrinogen level?",
+      "gold_sql": "SELECT COUNT(DISTINCT T1.ID) FROM Patient AS T1 INNER JOIN Laboratory AS T2 ON T1.ID = T2.ID WHERE T2.FG <= 150 OR T2.FG >= 450 AND T2.WBC > 3.5 AND T2.WBC < 9.0 AND T1.SEX = 'M'",
+      "baseline_pred": "SELECT COUNT(DISTINCT L.ID) FROM Patient P INNER JOIN Laboratory L ON P.ID = L.ID WHERE P.SEX = 'M' AND L.WBC > 3.5 AND L.WBC < 9.0 AND (L.FG <= 150 OR L.FG >= 450)",
+      "alt_pred": "SELECT COUNT(DISTINCT L.ID) FROM Patient P JOIN Laboratory L ON P.ID = L.ID WHERE P.SEX = 'M' AND L.WBC > 3.5 AND L.WBC < 9.0 AND (L.FG <= 150 OR L.FG >= 450)",
+      "alt_confidence": 0.98,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 26969.037499999104
+    },
+    {
+      "question_id": 1251,
+      "db_id": "thrombosis_prediction",
+      "difficulty": "simple",
+      "question": "How many patients with an Ig G higher than normal?",
+      "gold_sql": "SELECT COUNT(DISTINCT T1.ID) FROM Patient AS T1 INNER JOIN Laboratory AS T2 ON T1.ID = T2.ID INNER JOIN Examination AS T3 ON T3.ID = T2.ID WHERE T2.IGG >= 2000",
+      "baseline_pred": "SELECT COUNT(DISTINCT ID) FROM Laboratory WHERE IGG >= 2000",
+      "alt_pred": "SELECT COUNT(DISTINCT Lab.ID) FROM Laboratory AS Lab JOIN Patient AS P ON Lab.ID = P.ID WHERE Lab.IGG >= 2000",
+      "alt_confidence": 0.99,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 31754.1229999988
+    }
+  ]
+}

eval/reports/2026-05-17/groq-llama70b-on-v7.json

@@ -0,0 +1,235 @@
+{
+  "alt_model": "groq:llama-3.3-70b-versatile+grounded_critique+fewshot3",
+  "summary": {
+    "voted_better": 2,
+    "voted_worse": 0,
+    "voted_same": 13
+  },
+  "records": [
+    {
+      "question_id": 25,
+      "db_id": "california_schools",
+      "difficulty": "moderate",
+      "question": "Name schools in Riverside which the average of average math score for SAT is grater than 400, what is the funding type of these schools?",
+      "gold_sql": "SELECT T1.sname, T2.`Charter Funding Type` FROM satscores AS T1 INNER JOIN frpm AS T2 ON T1.cds = T2.CDSCode WHERE T2.`District Name` LIKE 'Riverside%' GROUP BY T1.sname, T2.`Charter Funding Type` HAVING CAST(SUM(T1.AvgScrMath) AS REAL) / COUNT(T1.cds) > 400",
+      "baseline_pred": "SELECT s.FundingType FROM schools AS s JOIN satscores AS t ON s.CDSCode = t.cds WHERE s.City = 'Riverside' AND t.AvgScrMath > 400 GROUP BY s.FundingType HAVING AVG(t.AvgScrMath) > 400",
+      "alt_pred": "SELECT T1.FundingType FROM schools AS T1 INNER JOIN satscores AS T2 ON T1.CDSCode = T2.cds WHERE T1.County = 'Riverside' GROUP BY T1.FundingType HAVING AVG(T2.AvgScrMath) > 400",
+      "alt_confidence": 0.8,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 4929.560500000662
+    },
+    {
+      "question_id": 37,
+      "db_id": "california_schools",
+      "difficulty": "moderate",
+      "question": "What is the complete address of the school with the lowest excellence rate? Indicate the Street, City, Zip and State.",
+      "gold_sql": "SELECT T2.Street, T2.City, T2.State, T2.Zip FROM satscores AS T1 INNER JOIN schools AS T2 ON T1.cds = T2.CDSCode ORDER BY CAST(T1.NumGE1500 AS REAL) / T1.NumTstTakr ASC LIMIT 1",
+      "baseline_pred": "SELECT Street, City, Zip, State FROM schools WHERE CDSCode = (SELECT cds FROM satscores WHERE NumTstTakr > 0 ORDER BY (NumGE1500 * 1.0 / NumTstTakr) ASC LIMIT 1)",
+      "alt_pred": "SELECT Street, City, Zip, State FROM schools ORDER BY (SELECT NumGE1500 / NumTstTakr FROM satscores WHERE satscores.cds = schools.CDSCode) ASC LIMIT 1",
+      "alt_confidence": 0.8,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 21531.478400000196
+    },
+    {
+      "question_id": 77,
+      "db_id": "california_schools",
+      "difficulty": "moderate",
+      "question": "Which schools served a grade span of Kindergarten to 9th grade in the county of Los Angeles and what is its Percent (%) Eligible FRPM (Ages 5-17)?",
+      "gold_sql": "SELECT T2.School, T1.`FRPM Count (Ages 5-17)` * 100 / T1.`Enrollment (Ages 5-17)` FROM frpm AS T1 INNER JOIN schools AS T2 ON T1.CDSCode = T2.CDSCode WHERE T2.County = 'Los Angeles' AND T2.GSserved = 'K-9'",
+      "baseline_pred": "SELECT f.`School Name`, (f.`FRPM Count (Ages 5-17)` / f.`Enrollment (Ages 5-17)`) * 100 AS `Percent (%) Eligible FRPM (Ages 5-17)` FROM frpm f JOIN schools s ON f.CDSCode = s.CDSCode WHERE f.`County Name` = 'Los Angeles' AND f.`Low Grade` = 'K' AND f.`High Grade` = '9'",
+      "alt_pred": "SELECT T2.School, (T1.`FRPM Count (Ages 5-17)` / T1.`Enrollment (Ages 5-17)`) * 100 FROM frpm AS T1 JOIN schools AS T2 ON T1.CDSCode = T2.CDSCode WHERE T2.County = 'Los Angeles' AND T1.`GSserved` = 'K-9'",
+      "alt_confidence": 0.9,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 43247.69329999981
+    },
+    {
+      "question_id": 125,
+      "db_id": "financial",
+      "difficulty": "challenging",
+      "question": "For loans contracts which are still running where client are in debt, list the district of the and the state the percentage unemployment rate increment from year 1995 to 1996.",
+      "gold_sql": "SELECT CAST((T3.A13 - T3.A12) AS REAL) * 100 / T3.A12 FROM loan AS T1 INNER JOIN account AS T2 ON T1.account_id = T2.account_id INNER JOIN district AS T3 ON T2.district_id = T3.district_id WHERE T1.status = 'D'",
+      "baseline_pred": "SELECT district.A2, CAST((district.A13 - district.A12) * 100.0 / district.A12 AS REAL) FROM loan JOIN account ON loan.account_id = account.account_id JOIN client ON account.district_id = client.district_id JOIN district ON client.district_id = district.district_id WHERE loan.status = 'D'",
+      "alt_pred": "SELECT T1.A2, ((T1.A13 - T1.A12) / T1.A12) * 100 FROM district AS T1 INNER JOIN client AS T2 ON T1.district_id = T2.district_id INNER JOIN disp AS T3 ON T2.client_id = T3.client_id INNER JOIN account AS T4 ON T3.account_id = T4.account_id INNER JOIN loan AS T5 ON T4.account_id = T5.account_id WHERE T5.status = 'D' GROUP BY T1.A2, T1.A12, T1.A13 HAVING COUNT(T5.loan_id) = (SELECT MAX(COUNT(loan_id)) FROM loan WHERE status = 'D' GROUP BY account_id)",
+      "alt_confidence": 0.8,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 32802.74060000011
+    },
+    {
+      "question_id": 173,
+      "db_id": "financial",
+      "difficulty": "challenging",
+      "question": "How often does account number 3 request an account statement to be released? What was the aim of debiting 3539 in total?",
+      "gold_sql": "SELECT T1.frequency, T2.k_symbol FROM account AS T1 INNER JOIN (SELECT account_id, k_symbol, SUM(amount) AS total_amount FROM `order` GROUP BY account_id, k_symbol) AS T2 ON T1.account_id = T2.account_id WHERE T1.account_id = 3 AND T2.total_amount = 3539",
+      "baseline_pred": "SELECT COUNT(*) AS frequency, k_symbol AS purpose FROM trans WHERE account_id = 3 AND type = 'VYDAJ' AND k_symbol = 'SLUZBY' GROUP BY k_symbol",
+      "alt_pred": "SELECT COUNT(T1.trans_id), T1.k_symbol FROM trans AS T1 INNER JOIN account AS T2 ON T1.account_id = T2.account_id WHERE T2.account_id = 3 AND T1.amount = 3539",
+      "alt_confidence": 0.8,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 16230.65299999871
+    },
+    {
+      "question_id": 207,
+      "db_id": "toxicology",
+      "difficulty": "challenging",
+      "question": "What elements are in a double type bond?",
+      "gold_sql": "SELECT DISTINCT T1.element FROM atom AS T1 INNER JOIN bond AS T2 ON T1.molecule_id = T2.molecule_id INNER JOIN connected AS T3 ON T1.atom_id = T3.atom_id WHERE T2.bond_type = '='",
+      "baseline_pred": "SELECT DISTINCT a.element FROM atom a JOIN bond b ON a.molecule_id = b.molecule_id JOIN connected c ON b.bond_id = c.bond_id WHERE b.bond_type = '='",
+      "alt_pred": "SELECT DISTINCT T1.element FROM atom AS T1 INNER JOIN connected AS T2 ON T1.atom_id = T2.atom_id INNER JOIN bond AS T3 ON T2.bond_id = T3.bond_id WHERE T3.bond_type = '='",
+      "alt_confidence": 0.9,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 5809.210999999777
+    },
+    {
+      "question_id": 219,
+      "db_id": "toxicology",
+      "difficulty": "challenging",
+      "question": "What is the percentage of carcinogenic molecules in triple type bonds?",
+      "gold_sql": "SELECT CAST(COUNT(DISTINCT CASE WHEN T2.label = '+' THEN T2.molecule_id ELSE NULL END) AS REAL) * 100 / COUNT(DISTINCT T2.molecule_id) FROM atom AS T1 INNER JOIN molecule AS T2 ON T1.molecule_id = T2.molecule_id INNER JOIN bond AS T3 ON T2.molecule_id = T3.molecule_id WHERE T3.bond_type = '#'",
+      "baseline_pred": "SELECT CAST(SUM(CASE WHEN b.bond_type = '#' THEN 1 ELSE 0 END) AS REAL) * 100.0 / COUNT(b.bond_id) FROM bond b INNER JOIN atom a ON b.molecule_id = a.molecule_id INNER JOIN molecule m ON a.molecule_id = m.molecule_id WHERE m.label = '+'",
+      "alt_pred": "SELECT CAST(SUM(CASE WHEN T1.bond_type = '#' THEN 1 ELSE 0 END) AS REAL) * 100 / COUNT(T1.bond_id) AS percent FROM bond AS T1 INNER JOIN molecule AS T2 ON T1.molecule_id = T2.molecule_id WHERE T2.label = '+'",
+      "alt_confidence": 0.9,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 5593.173399998705
+    },
+    {
+      "question_id": 349,
+      "db_id": "card_games",
+      "difficulty": "moderate",
+      "question": "Name the card and artist with the most ruling information. Also state if the card is a promotional printing.",
+      "gold_sql": "SELECT T1.name, T1.artist, T1.isPromo FROM cards AS T1 INNER JOIN rulings AS T2 ON T1.uuid = T2.uuid WHERE T1.isPromo = 1 AND T1.artist = (SELECT artist FROM cards WHERE isPromo = 1 GROUP BY artist HAVING COUNT(DISTINCT uuid) = (SELECT MAX(count_uuid) FROM ( SELECT COUNT(DISTINCT uuid) AS count_uuid FROM cards WHERE isPromo = 1 GROUP BY artist ))) LIMIT 1",
+      "baseline_pred": "SELECT c.name, c.artist, c.isPromo FROM cards AS c JOIN rulings AS r ON c.uuid = r.uuid GROUP BY c.uuid ORDER BY COUNT(r.id) DESC LIMIT 1",
+      "alt_pred": "SELECT T1.name, T1.artist, T1.isPromo FROM cards AS T1 INNER JOIN rulings AS T2 ON T1.uuid = T2.uuid GROUP BY T1.name, T1.artist, T1.isPromo ORDER BY COUNT(T2.uuid) DESC LIMIT 1",
+      "alt_confidence": 0.9,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 32800.355800000034
+    },
+    {
+      "question_id": 352,
+      "db_id": "card_games",
+      "difficulty": "moderate",
+      "question": "Calculate the percentage of the cards availabe in Chinese Simplified.",
+      "gold_sql": "SELECT CAST(SUM(CASE WHEN T2.language = 'Chinese Simplified' THEN 1 ELSE 0 END) AS REAL) * 100 / COUNT(T1.id) FROM cards AS T1 INNER JOIN foreign_data AS T2 ON T1.uuid = T2.uuid",
+      "baseline_pred": "SELECT CAST(COUNT(DISTINCT cards.uuid) AS REAL) * 100 / (SELECT COUNT(DISTINCT uuid) FROM cards) FROM cards JOIN foreign_data ON cards.uuid = foreign_data.uuid WHERE foreign_data.language = 'Chinese Simplified'",
+      "alt_pred": "SELECT CAST(SUM(CASE WHEN T2.language = 'Chinese Simplified' THEN 1 ELSE 0 END) AS REAL) * 100 / COUNT(T1.id) FROM cards AS T1 INNER JOIN foreign_data AS T2 ON T1.uuid = T2.uuid",
+      "alt_confidence": 0.8,
+      "baseline_match": false,
+      "alt_match": true,
+      "vote_match": true,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 30081.161299998712
+    },
+    {
+      "question_id": 358,
+      "db_id": "card_games",
+      "difficulty": "simple",
+      "question": "What is the border color of card \"Ancestor's Chosen\"?",
+      "gold_sql": "SELECT DISTINCT borderColor FROM cards WHERE name = 'Ancestor''s Chosen'",
+      "baseline_pred": "SELECT borderColor FROM cards WHERE name = 'Ancestor''s Chosen'",
+      "alt_pred": "SELECT borderColor FROM cards WHERE name = 'Ancestor''s Chosen'",
+      "alt_confidence": 1.0,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 44694.91830000152
+    },
+    {
+      "question_id": 366,
+      "db_id": "card_games",
+      "difficulty": "simple",
+      "question": "What is the rule of playing card \"Benalish Knight\"?",
+      "gold_sql": "SELECT T2.format FROM cards AS T1 INNER JOIN legalities AS T2 ON T1.uuid = T2.uuid WHERE T1.name = 'Benalish Knight'",
+      "baseline_pred": "SELECT text FROM rulings WHERE uuid IN (SELECT uuid FROM cards WHERE name LIKE '%Benalish Knight%')",
+      "alt_pred": "SELECT T2.format FROM cards AS T1 INNER JOIN legalities AS T2 ON T1.uuid = T2.uuid WHERE T1.name = 'Benalish Knight'",
+      "alt_confidence": 0.8,
+      "baseline_match": false,
+      "alt_match": true,
+      "vote_match": true,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 22241.72820000058
+    },
+    {
+      "question_id": 407,
+      "db_id": "card_games",
+      "difficulty": "moderate",
+      "question": "Lists all types of cards in German.",
+      "gold_sql": "SELECT T1.subtypes, T1.supertypes FROM cards AS T1 INNER JOIN foreign_data AS T2 ON T1.uuid = T2.uuid WHERE T2.language = 'German' AND T1.subtypes IS NOT NULL AND T1.supertypes IS NOT NULL",
+      "baseline_pred": "SELECT DISTINCT T1.subtypes, T1.supertypes FROM cards AS T1 INNER JOIN foreign_data AS T2 ON T1.uuid = T2.uuid WHERE T2.language = 'German' AND T1.subtypes IS NOT NULL AND T1.supertypes IS NOT NULL",
+      "alt_pred": "SELECT DISTINCT T1.subtypes, T1.supertypes FROM cards AS T1 INNER JOIN foreign_data AS T2 ON T1.uuid = T2.uuid WHERE T2.language = 'German' AND T1.subtypes IS NOT NULL AND T1.supertypes IS NOT NULL",
+      "alt_confidence": 0.8,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 35438.194599999406
+    },
+    {
+      "question_id": 408,
+      "db_id": "card_games",
+      "difficulty": "moderate",
+      "question": "How many unknown power cards contain info about the triggered ability",
+      "gold_sql": "SELECT Count(DISTINCT T1.id) FROM cards AS T1 INNER JOIN rulings AS T2 ON T1.uuid = T2.uuid WHERE (T1.power IS NULL OR T1.power = '*') AND T2.text LIKE '%triggered ability%'",
+      "baseline_pred": "SELECT COUNT(*) FROM cards WHERE (power IS NULL OR power = '*') AND text LIKE '%triggered ability%'",
+      "alt_pred": "SELECT COUNT(*) FROM cards WHERE (power IS NULL OR power = '*') AND text LIKE '%triggered ability%'",
+      "alt_confidence": 0.9,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 29518.018900000243
+    },
+    {
+      "question_id": 484,
+      "db_id": "card_games",
+      "difficulty": "moderate",
+      "question": "Please list the Italian names of the cards in the set Coldsnap with the highest converted mana cost.",
+      "gold_sql": "SELECT T2.name FROM foreign_data AS T1 INNER JOIN cards AS T2 ON T2.uuid = T1.uuid INNER JOIN sets AS T3 ON T3.code = T2.setCode WHERE T3.name = 'Coldsnap' AND T1.language = 'Italian' ORDER BY T2.convertedManaCost DESC",
+      "baseline_pred": "SELECT fd.name FROM cards AS c JOIN foreign_data AS fd ON c.uuid = fd.uuid JOIN sets AS s ON c.setCode = s.code WHERE s.name = 'Coldsnap' AND fd.language = 'Italian' ORDER BY c.convertedManaCost DESC LIMIT 1",
+      "alt_pred": "SELECT T2.name FROM sets AS T1 INNER JOIN cards AS T2 ON T1.code = T2.setCode INNER JOIN set_translations AS T3 ON T1.code = T3.setCode WHERE T1.name = 'Coldsnap' AND T3.language = 'Italian' ORDER BY T2.convertedManaCost DESC LIMIT 1",
+      "alt_confidence": 0.9,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 30588.251799999853
+    },
+    {
+      "question_id": 743,
+      "db_id": "superhero",
+      "difficulty": "challenging",
+      "question": "What is the percentage of superheroes who act in their own self-interest or make decisions based on their own moral code? Indicate how many of the said superheroes were published by Marvel Comics.",
+      "gold_sql": "SELECT (CAST(COUNT(*) AS REAL) * 100 / (SELECT COUNT(*) FROM superhero)), CAST(SUM(CASE WHEN T2.publisher_name = 'Marvel Comics' THEN 1 ELSE 0 END) AS REAL) FROM superhero AS T1 INNER JOIN publisher AS T2 ON T1.publisher_id = T2.id INNER JOIN alignment AS T3 ON T3.id = T1.alignment_id WHERE T3.alignment = 'Bad'",
+      "baseline_pred": "SELECT CAST(SUM(CASE WHEN a.alignment = 'Bad' THEN 1 ELSE 0 END) AS REAL) * 100 / COUNT(s.id) AS percentage, SUM(CASE WHEN a.alignment = 'Bad' AND p.publisher_name = 'Marvel Comics' THEN 1 ELSE 0 END) FROM superhero s INNER JOIN alignment a ON s.alignment_id = a.id LEFT JOIN publisher p ON s.publisher_id = p.id",
+      "alt_pred": "SELECT CAST(SUM(CASE WHEN T1.alignment_id = 1 THEN 1 ELSE 0 END) AS REAL) * 100 / COUNT(T1.id) AS percentage, SUM(CASE WHEN T1.publisher_id = (SELECT id FROM publisher WHERE publisher_name = 'Marvel Comics') AND T1.alignment_id = 1 THEN 1 ELSE 0 END) AS marvel_count FROM superhero AS T1",
+      "alt_confidence": 0.9,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 7933.113900000535
+    }
+  ]
+}

eval/reports/2026-05-17/groq-qwen3-throttled-on-v7.json

@@ -0,0 +1,85 @@
+{
+  "alt_model": "groq:qwen/qwen3-32b+grounded_critique+fewshot3",
+  "summary": {
+    "voted_better": 1,
+    "voted_worse": 0,
+    "voted_same": 4
+  },
+  "records": [
+    {
+      "question_id": 219,
+      "db_id": "toxicology",
+      "difficulty": "challenging",
+      "question": "What is the percentage of carcinogenic molecules in triple type bonds?",
+      "gold_sql": "SELECT CAST(COUNT(DISTINCT CASE WHEN T2.label = '+' THEN T2.molecule_id ELSE NULL END) AS REAL) * 100 / COUNT(DISTINCT T2.molecule_id) FROM atom AS T1 INNER JOIN molecule AS T2 ON T1.molecule_id = T2.molecule_id INNER JOIN bond AS T3 ON T2.molecule_id = T3.molecule_id WHERE T3.bond_type = '#'",
+      "baseline_pred": "SELECT CAST(SUM(CASE WHEN b.bond_type = '#' THEN 1 ELSE 0 END) AS REAL) * 100.0 / COUNT(b.bond_id) FROM bond b INNER JOIN atom a ON b.molecule_id = a.molecule_id INNER JOIN molecule m ON a.molecule_id = m.molecule_id WHERE m.label = '+'",
+      "alt_pred": "SELECT (COUNT(DISTINCT CASE WHEN m.label = '+' THEN m.molecule_id END) * 100.0 / COUNT(DISTINCT m.molecule_id)) FROM bond AS b JOIN molecule AS m ON b.molecule_id = m.molecule_id WHERE b.bond_type = '#'",
+      "alt_confidence": 0.95,
+      "baseline_match": false,
+      "alt_match": true,
+      "vote_match": true,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 30143.128100000467
+    },
+    {
+      "question_id": 1247,
+      "db_id": "thrombosis_prediction",
+      "difficulty": "challenging",
+      "question": "Among the male patients who have a normal level of white blood cells, how many of them have an abnormal fibrinogen level?",
+      "gold_sql": "SELECT COUNT(DISTINCT T1.ID) FROM Patient AS T1 INNER JOIN Laboratory AS T2 ON T1.ID = T2.ID WHERE T2.FG <= 150 OR T2.FG >= 450 AND T2.WBC > 3.5 AND T2.WBC < 9.0 AND T1.SEX = 'M'",
+      "baseline_pred": "SELECT COUNT(DISTINCT L.ID) FROM Patient P INNER JOIN Laboratory L ON P.ID = L.ID WHERE P.SEX = 'M' AND L.WBC > 3.5 AND L.WBC < 9.0 AND (L.FG <= 150 OR L.FG >= 450)",
+      "alt_pred": "SELECT COUNT(DISTINCT Patient.ID) FROM Patient JOIN Laboratory ON Patient.ID = Laboratory.ID WHERE Patient.SEX = 'M' AND `WBC` > 3.5 AND `WBC` < 9.0 AND (`FG` <= 150 OR `FG` >= 450)",
+      "alt_confidence": 0.95,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 53296.70189999888
+    },
+    {
+      "question_id": 1251,
+      "db_id": "thrombosis_prediction",
+      "difficulty": "simple",
+      "question": "How many patients with an Ig G higher than normal?",
+      "gold_sql": "SELECT COUNT(DISTINCT T1.ID) FROM Patient AS T1 INNER JOIN Laboratory AS T2 ON T1.ID = T2.ID INNER JOIN Examination AS T3 ON T3.ID = T2.ID WHERE T2.IGG >= 2000",
+      "baseline_pred": "SELECT COUNT(DISTINCT ID) FROM Laboratory WHERE IGG >= 2000",
+      "alt_pred": "SELECT COUNT(DISTINCT ID) FROM Laboratory WHERE IGG >= 2000",
+      "alt_confidence": 0.95,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 38587.686400000166
+    },
+    {
+      "question_id": 1275,
+      "db_id": "thrombosis_prediction",
+      "difficulty": "moderate",
+      "question": "Among the patients who has a normal level of anti-centromere and a normal level of anti-SSB, how many of them are male?",
+      "gold_sql": "SELECT COUNT(DISTINCT T1.ID) FROM Patient AS T1 INNER JOIN Laboratory AS T2 ON T1.ID = T2.ID WHERE T2.CENTROMEA IN ('negative', '0') AND T2.SSB IN ('negative', '0') AND T1.SEX = 'M'",
+      "baseline_pred": "SELECT COUNT(DISTINCT P.ID) FROM Patient P JOIN Examination E ON P.ID = E.ID WHERE E.CENTROMEA IN ('-', '+-') AND E.SSB IN ('-', '+-') AND P.SEX = 'M'",
+      "alt_pred": "SELECT COUNT(DISTINCT T1.ID) FROM Patient AS T1 INNER JOIN Laboratory AS T2 ON T1.ID = T2.ID WHERE T2.CENTROMEA IN ('-', '+-') AND T2.SSB IN ('-', '+-') AND T1.SEX = 'M'",
+      "alt_confidence": 0.95,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 50812.23180000052
+    },
+    {
+      "question_id": 1399,
+      "db_id": "student_club",
+      "difficulty": "moderate",
+      "question": "Did Maya Mclean attend the 'Women's Soccer' event?",
+      "gold_sql": "SELECT CASE WHEN T3.event_name = 'Women''s Soccer' THEN 'YES' END AS result FROM member AS T1 INNER JOIN attendance AS T2 ON T1.member_id = T2.link_to_member INNER JOIN event AS T3 ON T2.link_to_event = T3.event_id WHERE T1.first_name = 'Maya' AND T1.last_name = 'Mclean'",
+      "baseline_pred": "SELECT COUNT(*) > 0 AS attended FROM attendance AS a JOIN member AS m ON a.link_to_member = m.member_id JOIN event AS e ON a.link_to_event = e.event_id WHERE m.first_name = 'Maya' AND m.last_name = 'Mclean' AND e.event_name = 'Women''s Soccer'",
+      "alt_pred": "SELECT event.event_name FROM member INNER JOIN attendance ON member.member_id = attendance.link_to_member INNER JOIN event ON attendance.link_to_event = event.event_id WHERE member.first_name = 'Maya' AND member.last_name = 'Mclean' AND event.event_name = 'Women''s Soccer'",
+      "alt_confidence": 1.0,
+      "baseline_match": false,
+      "alt_match": false,
+      "vote_match": false,
+      "vote_source": "critique-retry",
+      "elapsed_ms": 42464.109099999405
+    }
+  ]
+}

scripts/run_critique_retry.py

@@ -31,6 +31,7 @@ from nl_sql.eval.dataset import load_bird_mini_dev
 from nl_sql.eval.metrics.execution_accuracy import compare_results
 from nl_sql.eval.runner import _compose_question, _execute_gold
 from nl_sql.llm.cache import CachingEmbeddingProvider, CachingLLMProvider
+from nl_sql.llm.providers.groq import GroqProvider
 from nl_sql.llm.providers.mistral import MistralProvider
 from nl_sql.schema_index.indexer import SchemaIndex
 
@@ -48,6 +49,28 @@ def main() -> int:
         help="PipelineConfig.fewshot_top_k (default 3 = G prod). "
         "Use 5 for P2.B selective expansion experiment.",
     )
+    p.add_argument(
+        "--gen-model",
+        type=str,
+        default="codestral-latest",
+        help="Mistral gen model id (default codestral-latest = G prod). "
+        "Use mistral-large-latest for cross-model voting on residue.",
+    )
+    p.add_argument(
+        "--sleep-between",
+        type=float,
+        default=0.0,
+        help="Sleep N seconds between cases — required for mistral-large "
+        "on free tier (rate-limited ~2 req/s).",
+    )
+    p.add_argument(
+        "--provider",
+        type=str,
+        choices=("mistral", "groq"),
+        default="mistral",
+        help="SQL provider: mistral (default, uses --gen-model) or groq "
+        "(uses --gen-model as Groq model id, e.g. qwen/qwen3-32b).",
+    )
     args = p.parse_args()
 
     settings = get_settings()
@@ -59,8 +82,11 @@ def main() -> int:
     examples = {e.question_id: e for e in load_bird_mini_dev(args.bird_root)}
     registry = get_default_registry()
 
-    mistral = MistralProvider(api_key=settings.mistral_api_key, gen_model="codestral-latest")
-    sql_prov = CachingLLMProvider(mistral, cache_dir=settings.llm_cache_dir)
+    if args.provider == "mistral":
+        gen_provider = MistralProvider(api_key=settings.mistral_api_key, gen_model=args.gen_model)
+    else:
+        gen_provider = GroqProvider(api_key=settings.groq_api_key, model=args.gen_model)
+    sql_prov = CachingLLMProvider(gen_provider, cache_dir=settings.llm_cache_dir)
     expl_prov = sql_prov  # same provider for explain
     emb = CachingEmbeddingProvider(
         MistralProvider(api_key=settings.mistral_api_key), cache_dir=settings.llm_cache_dir
@@ -155,6 +181,8 @@ def main() -> int:
             )
         finally:
             engine.dispose()
+        if args.sleep_between > 0:
+            time.sleep(args.sleep_between)
 
     print("\n=== critique-retry summary ===", file=sys.stderr)
     print(f"  cases: {len(records)}", file=sys.stderr)
@@ -166,7 +194,7 @@ def main() -> int:
     args.out.write_text(
         json.dumps(
             {
-                "alt_model": "codestral+grounded_critique",
+                "alt_model": f"{args.provider}:{args.gen_model}+grounded_critique+fewshot{args.fewshot_top_k}",
                 "summary": {
                     "voted_better": rescued,
                     "voted_worse": regressed,