"""Schema Recall@k — secondary metric per docs/03_eval_methodology.md §1.2. For each example we know which tables the gold SQL touches (`gold_tables`). After retrieval we know which tables the context_builder shortlisted (`retrieved_tables`). Recall@k is the share of examples where every gold table is present in the top-k retrieval set. Why "all-or-nothing" rather than per-table fraction: a SQL referencing 3 tables is broken if any one is missing — the LLM cannot recover. """ from __future__ import annotations from collections.abc import Sequence def schema_recall_at_k( gold_tables: Sequence[str], retrieved_tables: Sequence[str], *, case_insensitive: bool = True, ) -> bool: """True iff every `gold_table` appears in `retrieved_tables`. SQLite identifiers are case-insensitive; BIRD gold + introspector both preserve original case but the comparison should be case-insensitive by default. Pass `case_insensitive=False` to enforce exact match. """ if not gold_tables: return True if case_insensitive: retrieved = {t.lower() for t in retrieved_tables} return all(g.lower() in retrieved for g in gold_tables) retrieved_set = set(retrieved_tables) return all(g in retrieved_set for g in gold_tables)