File size: 3,707 Bytes
d54f61e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
"""Demo core, no UI and no GPU: score a final query on an item's original DB and its counterfactual DBs, with
the reward each function gives. demo/index.html runs this module in the browser (Pyodide); the api_* functions
return JSON strings for it. Needs data/test (items + DBs) and results/traj_*.jsonl.
"""
import json
from collections import defaultdict
from pathlib import Path

from .rewards import REWARDS, _exec, final_sql

ROOT = Path(__file__).resolve().parent.parent
ITEMS = {x["id"]: x for x in map(json.loads, open(ROOT / "data/test/items.jsonl"))}
TRAJ = defaultdict(dict)  # item id -> {"model #sample": messages}
for p in sorted((ROOT / "results").glob("traj_*.jsonl")):
    for r in map(json.loads, open(p)):
        if r["id"] in ITEMS:
            TRAJ[r["id"]][f'{r["model"]} #{r.get("sample", 0)}'] = r["messages"]


def _reward(name, item, completion, dbs=None, gold=None):
    return REWARDS[name]([completion], dbs=[dbs or item["dbs"]], gold=[gold or item["gold"]], ordered=[item["ordered"]])[0]


def _blind_run(i):
    """First recorded run of item i that passes the original DB but fails a counterfactual, else None."""
    for run, msgs in sorted(TRAJ[i].items()):
        if _reward("R_exec_rex", ITEMS[i], msgs) and not _reward("R_cf_strict", ITEMS[i], msgs):
            return run


BLIND = {i: _blind_run(i) for i in ITEMS}
ORDER = sorted(ITEMS, key=lambda i: (BLIND[i] is None, i))  # showcase items first


def _short(rows, n=3):
    if rows is None:
        return "SQL error"
    s = "; ".join(", ".join(map(str, r)) for r in rows[:n])
    return s + (f" ... ({len(rows)} rows)" if len(rows) > n else "")


def score(item_id, sql):
    item = ITEMS[item_id]
    sql = (sql or "").strip().rstrip(";")
    comp = f"```sql\n{sql}\n```" if sql else ""
    edits = ["original (agent saw this)"] + [f"{e} ({k})" for e, k in zip(item["cf_edits"], item["cf_kinds"])]
    checks = []
    for db, gold, edit in zip(item["dbs"], item["gold"], edits):
        got = _exec(db, sql) if sql else None
        checks.append({"db": edit, "gold": _short([tuple(r) for r in gold]), "got": _short(got),
                       "ok": bool(_reward("R_cf_strict", item, comp, [db], [gold]))})
    return {"checks": checks, "rewards": {n: _reward(n, item, comp) for n in REWARDS}}


def item_view(item_id):
    item = ITEMS[item_id]
    runs = sorted(TRAJ[item_id])
    return {"family": item["family"], "template": item["template"], "question": item["question"],
            "db": item["dbs"][0].split("/")[-2], "gold_sql": item["gold_sql"], "runs": runs,
            "default": BLIND[item_id] or (runs[0] if runs else None)}


def run_view(item_id, run):
    msgs = TRAJ[item_id].get(run or "", [])
    return {"messages": [{"role": m["role"], "content": m.get("content") or ""} for m in msgs], "sql": final_sql(msgs) or ""}


def api_items():
    return json.dumps([{"id": i, "blind": BLIND[i] is not None} for i in ORDER])


def api_item(item_id):
    return json.dumps(item_view(item_id))


def api_run(item_id, run):
    return json.dumps(run_view(item_id, run))


def api_score(item_id, sql):
    return json.dumps(score(item_id, sql))


if __name__ == "__main__":  # self-check: a showcase item's blind run, gold SQL, junk SQL
    i = ORDER[0]
    v = item_view(i)
    s = score(i, run_view(i, v["default"])["sql"])
    assert s["rewards"]["R_exec_rex"] == 1.0 and s["rewards"]["R_cf_strict"] == 0.0, s
    assert all(x == 1.0 for x in score(i, v["gold_sql"])["rewards"].values())
    assert all(x == 0.0 for x in score(i, "SELEC nonsense")["rewards"].values())
    print("ok", len(ITEMS), "items,", sum(b is not None for b in BLIND.values()), "showcase;", i, v["default"], s["rewards"])