LogicPalette's picture
demo update
d54f61e verified
Raw History Blame Contribute Delete
3.71 kB
"""Demo core, no UI and no GPU: score a final query on an item's original DB and its counterfactual DBs, with
the reward each function gives. demo/index.html runs this module in the browser (Pyodide); the api_* functions
return JSON strings for it. Needs data/test (items + DBs) and results/traj_*.jsonl.
"""
import json
from collections import defaultdict
from pathlib import Path
from .rewards import REWARDS, _exec, final_sql
ROOT = Path(__file__).resolve().parent.parent
ITEMS = {x["id"]: x for x in map(json.loads, open(ROOT / "data/test/items.jsonl"))}
TRAJ = defaultdict(dict) # item id -> {"model #sample": messages}
for p in sorted((ROOT / "results").glob("traj_*.jsonl")):
for r in map(json.loads, open(p)):
if r["id"] in ITEMS:
TRAJ[r["id"]][f'{r["model"]} #{r.get("sample", 0)}'] = r["messages"]
def _reward(name, item, completion, dbs=None, gold=None):
return REWARDS[name]([completion], dbs=[dbs or item["dbs"]], gold=[gold or item["gold"]], ordered=[item["ordered"]])[0]
def _blind_run(i):
"""First recorded run of item i that passes the original DB but fails a counterfactual, else None."""
for run, msgs in sorted(TRAJ[i].items()):
if _reward("R_exec_rex", ITEMS[i], msgs) and not _reward("R_cf_strict", ITEMS[i], msgs):
return run
BLIND = {i: _blind_run(i) for i in ITEMS}
ORDER = sorted(ITEMS, key=lambda i: (BLIND[i] is None, i)) # showcase items first
def _short(rows, n=3):
if rows is None:
return "SQL error"
s = "; ".join(", ".join(map(str, r)) for r in rows[:n])
return s + (f" ... ({len(rows)} rows)" if len(rows) > n else "")
def score(item_id, sql):
item = ITEMS[item_id]
sql = (sql or "").strip().rstrip(";")
comp = f"```sql\n{sql}\n```" if sql else ""
edits = ["original (agent saw this)"] + [f"{e} ({k})" for e, k in zip(item["cf_edits"], item["cf_kinds"])]
checks = []
for db, gold, edit in zip(item["dbs"], item["gold"], edits):
got = _exec(db, sql) if sql else None
checks.append({"db": edit, "gold": _short([tuple(r) for r in gold]), "got": _short(got),
"ok": bool(_reward("R_cf_strict", item, comp, [db], [gold]))})
return {"checks": checks, "rewards": {n: _reward(n, item, comp) for n in REWARDS}}
def item_view(item_id):
item = ITEMS[item_id]
runs = sorted(TRAJ[item_id])
return {"family": item["family"], "template": item["template"], "question": item["question"],
"db": item["dbs"][0].split("/")[-2], "gold_sql": item["gold_sql"], "runs": runs,
"default": BLIND[item_id] or (runs[0] if runs else None)}
def run_view(item_id, run):
msgs = TRAJ[item_id].get(run or "", [])
return {"messages": [{"role": m["role"], "content": m.get("content") or ""} for m in msgs], "sql": final_sql(msgs) or ""}
def api_items():
return json.dumps([{"id": i, "blind": BLIND[i] is not None} for i in ORDER])
def api_item(item_id):
return json.dumps(item_view(item_id))
def api_run(item_id, run):
return json.dumps(run_view(item_id, run))
def api_score(item_id, sql):
return json.dumps(score(item_id, sql))
if __name__ == "__main__": # self-check: a showcase item's blind run, gold SQL, junk SQL
i = ORDER[0]
v = item_view(i)
s = score(i, run_view(i, v["default"])["sql"])
assert s["rewards"]["R_exec_rex"] == 1.0 and s["rewards"]["R_cf_strict"] == 0.0, s
assert all(x == 1.0 for x in score(i, v["gold_sql"])["rewards"].values())
assert all(x == 0.0 for x in score(i, "SELEC nonsense")["rewards"].values())
print("ok", len(ITEMS), "items,", sum(b is not None for b in BLIND.values()), "showcase;", i, v["default"], s["rewards"])