Download sqlcf/demo.py from sqlcf-lexsi/counterfactual-sql-demo: direct link, hf CLI and curl.
- Browser
- Download file 3.71 kB
-
https://huggingface.co/spaces/sqlcf-lexsi/counterfactual-sql-demo/resolve/main/sqlcf/demo.py
- Command line
-
hf download hf://spaces/sqlcf-lexsi/counterfactual-sql-demo/sqlcf/demo.py
-
curl -L -o demo.py https://huggingface.co/spaces/sqlcf-lexsi/counterfactual-sql-demo/resolve/main/sqlcf/demo.py
3.71 kB
| """Demo core, no UI and no GPU: score a final query on an item's original DB and its counterfactual DBs, with | |
| the reward each function gives. demo/index.html runs this module in the browser (Pyodide); the api_* functions | |
| return JSON strings for it. Needs data/test (items + DBs) and results/traj_*.jsonl. | |
| """ | |
| import json | |
| from collections import defaultdict | |
| from pathlib import Path | |
| from .rewards import REWARDS, _exec, final_sql | |
| ROOT = Path(__file__).resolve().parent.parent | |
| ITEMS = {x["id"]: x for x in map(json.loads, open(ROOT / "data/test/items.jsonl"))} | |
| TRAJ = defaultdict(dict) # item id -> {"model #sample": messages} | |
| for p in sorted((ROOT / "results").glob("traj_*.jsonl")): | |
| for r in map(json.loads, open(p)): | |
| if r["id"] in ITEMS: | |
| TRAJ[r["id"]][f'{r["model"]} #{r.get("sample", 0)}'] = r["messages"] | |
| def _reward(name, item, completion, dbs=None, gold=None): | |
| return REWARDS[name]([completion], dbs=[dbs or item["dbs"]], gold=[gold or item["gold"]], ordered=[item["ordered"]])[0] | |
| def _blind_run(i): | |
| """First recorded run of item i that passes the original DB but fails a counterfactual, else None.""" | |
| for run, msgs in sorted(TRAJ[i].items()): | |
| if _reward("R_exec_rex", ITEMS[i], msgs) and not _reward("R_cf_strict", ITEMS[i], msgs): | |
| return run | |
| BLIND = {i: _blind_run(i) for i in ITEMS} | |
| ORDER = sorted(ITEMS, key=lambda i: (BLIND[i] is None, i)) # showcase items first | |
| def _short(rows, n=3): | |
| if rows is None: | |
| return "SQL error" | |
| s = "; ".join(", ".join(map(str, r)) for r in rows[:n]) | |
| return s + (f" ... ({len(rows)} rows)" if len(rows) > n else "") | |
| def score(item_id, sql): | |
| item = ITEMS[item_id] | |
| sql = (sql or "").strip().rstrip(";") | |
| comp = f"```sql\n{sql}\n```" if sql else "" | |
| edits = ["original (agent saw this)"] + [f"{e} ({k})" for e, k in zip(item["cf_edits"], item["cf_kinds"])] | |
| checks = [] | |
| for db, gold, edit in zip(item["dbs"], item["gold"], edits): | |
| got = _exec(db, sql) if sql else None | |
| checks.append({"db": edit, "gold": _short([tuple(r) for r in gold]), "got": _short(got), | |
| "ok": bool(_reward("R_cf_strict", item, comp, [db], [gold]))}) | |
| return {"checks": checks, "rewards": {n: _reward(n, item, comp) for n in REWARDS}} | |
| def item_view(item_id): | |
| item = ITEMS[item_id] | |
| runs = sorted(TRAJ[item_id]) | |
| return {"family": item["family"], "template": item["template"], "question": item["question"], | |
| "db": item["dbs"][0].split("/")[-2], "gold_sql": item["gold_sql"], "runs": runs, | |
| "default": BLIND[item_id] or (runs[0] if runs else None)} | |
| def run_view(item_id, run): | |
| msgs = TRAJ[item_id].get(run or "", []) | |
| return {"messages": [{"role": m["role"], "content": m.get("content") or ""} for m in msgs], "sql": final_sql(msgs) or ""} | |
| def api_items(): | |
| return json.dumps([{"id": i, "blind": BLIND[i] is not None} for i in ORDER]) | |
| def api_item(item_id): | |
| return json.dumps(item_view(item_id)) | |
| def api_run(item_id, run): | |
| return json.dumps(run_view(item_id, run)) | |
| def api_score(item_id, sql): | |
| return json.dumps(score(item_id, sql)) | |
| if __name__ == "__main__": # self-check: a showcase item's blind run, gold SQL, junk SQL | |
| i = ORDER[0] | |
| v = item_view(i) | |
| s = score(i, run_view(i, v["default"])["sql"]) | |
| assert s["rewards"]["R_exec_rex"] == 1.0 and s["rewards"]["R_cf_strict"] == 0.0, s | |
| assert all(x == 1.0 for x in score(i, v["gold_sql"])["rewards"].values()) | |
| assert all(x == 0.0 for x in score(i, "SELEC nonsense")["rewards"].values()) | |
| print("ok", len(ITEMS), "items,", sum(b is not None for b in BLIND.values()), "showcase;", i, v["default"], s["rewards"]) | |