File size: 6,724 Bytes
b77d5ed | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 | """Play many full games in parallel on GPU with the pure policy (one forward pass
per move, no search). Watches a checkpoint and re-evaluates whenever it changes.
python -m reflex.evaluate --ckpt runs/reflex/latest.pt --games 1024 [--watch] [--sym]
"""
import argparse, json, os, time
import torch
import torch.nn.functional as F
from .env import action_features, reset, score_of, spawn, sym, sym_perm
from .model import load
TILES = [2048, 4096, 8192, 16384, 32768, 65536]
@torch.no_grad()
def policy(model, b, use_sym=False):
feats, legal, after, reward = action_features(b)
with torch.autocast("cuda", dtype=torch.bfloat16, enabled=b.is_cuda):
logits, _ = model(b, feats, legal)
p = F.softmax(logits, -1)
if use_sym: # average the distribution over the 8 symmetric views (still one batched pass)
P = sym_perm().to(b.device)
bs = torch.cat([sym(b, k) for k in range(1, 8)])
f2, l2, _, _ = action_features(bs)
with torch.autocast("cuda", dtype=torch.bfloat16, enabled=b.is_cuda):
lg, _ = model(bs, f2, l2)
p2 = F.softmax(lg, -1).view(7, len(b), 4)
for k in range(1, 8):
p = p + p2[k - 1][:, P[k]]
p = p / 8
return p.nan_to_num(0), legal, after, reward
@torch.no_grad()
def lookahead(model, b, chunk=65536):
"""1-ply expectimax on the value head: Q(s,a) = r_a + E[V(s')] with the expectation exact
over every spawn (a 2 w.p. .9 or a 4 w.p. .1 in each empty cell of the afterstate).
Not search-free: about 8x(empty cells) forward passes per move. Returns q (n,4), -inf if illegal."""
_, legal, after, reward = action_features(b)
flat = after.flatten(2) # (n,4,16)
empty = flat == 0
eye = torch.eye(16, dtype=flat.dtype, device=b.device)
child = flat[:, :, None, None] + eye[:, None] * torch.tensor([1, 2], device=b.device)[:, None] # (n,4,16,2,16)
w = empty[..., None] * torch.tensor([.9, .1], device=b.device) / empty.sum(-1, keepdim=True)[..., None].clamp(min=1)
valid = (w > 0) & legal[:, :, None, None]
cb = child[valid].view(-1, 4, 4)
vals = []
for i in range(0, len(cb), chunk):
f, l, _, _ = action_features(cb[i:i + chunk])
with torch.autocast("cuda", dtype=torch.bfloat16, enabled=b.is_cuda):
_, v = model(cb[i:i + chunk], f, l)
vals.append(torch.expm1(10 * v.float()).clamp(min=0)) # value head predicts log1p(return)/10
ev = torch.zeros(valid.shape, device=b.device)
ev[valid] = torch.cat(vals) if vals else ev[valid]
q = reward.float() + (ev * w).sum((2, 3))
return q.masked_fill(~legal, float("-inf")), legal, after, reward
@torch.no_grad()
def play(model, n, device="cuda", use_sym=False, live=None, seed=0, ply1=False):
gen = torch.Generator(device=device).manual_seed(seed)
b = reset(n, device, gen)
score = torch.zeros(n, dtype=torch.int64, device=device)
moves = torch.zeros(n, dtype=torch.int64, device=device)
alive = torch.ones(n, dtype=torch.bool, device=device)
t0, steps = time.time(), 0
while alive.any():
idx = alive.nonzero().squeeze(1)
if ply1:
p, legal, after, reward = lookahead(model, b[idx])
else:
p, legal, after, reward = policy(model, b[idx], use_sym)
dead = ~legal.any(1)
a = p.argmax(1)
nb = after[torch.arange(len(idx)), a]
r = reward[torch.arange(len(idx)), a]
live_idx = idx[~dead]
b[live_idx] = spawn(nb[~dead], gen)
score[live_idx] += r[~dead]
moves[live_idx] += 1
alive[idx[dead]] = False
steps += 1
if live and steps % 25 == 0:
j = int(score.argmax())
json.dump(dict(board=b[j].tolist(), score=int(score[j]), moves=int(moves[j]),
alive=int(alive.sum()), games=n, step=steps), open(live + ".tmp", "w"))
os.replace(live + ".tmp", live)
dt = time.time() - t0
mx = b.flatten(1).max(1).values
res = dict(games=n, mean_score=score.float().mean().item(), median_score=score.float().median().item(),
max_score=int(score.max()), mean_moves=moves.float().mean().item(),
moves_per_s=moves.sum().item() / dt, seconds=dt, sym=use_sym, ply1=ply1)
for t in TILES:
res[f"rate_{t}"] = (mx >= t.bit_length() - 1).float().mean().item()
best = int(score.argmax())
res["best_board"] = b[best].tolist()
return res
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--ckpt", default="/root/runs/reflex/latest.pt")
ap.add_argument("--games", type=int, default=1024)
ap.add_argument("--sym", action="store_true")
ap.add_argument("--ply1", action="store_true", help="1-ply expectimax on the value head (not search-free)")
ap.add_argument("--watch", action="store_true")
ap.add_argument("--out", default="/root/runs/eval.jsonl")
ap.add_argument("--seed", type=int, default=0)
ap.add_argument("--min-interval", type=float, default=0, help="watch mode: seconds between evaluations "
"(evaluation shares the GPU with training)")
ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
args = ap.parse_args()
last, best = None, -1
while True:
if os.path.exists(args.ckpt) and os.path.getmtime(args.ckpt) != last:
last, last_run = os.path.getmtime(args.ckpt), time.time()
model = load(args.ckpt, args.device)
step = torch.load(args.ckpt, weights_only=False, map_location="cpu").get("step", 0)
res = play(model, args.games, device=args.device, use_sym=args.sym, seed=args.seed, ply1=args.ply1,
live=os.path.join(os.path.dirname(args.out), "live.json"))
res.update(t=time.time(), step=step, ckpt=args.ckpt)
if args.watch and res["mean_score"] > best: # keep the best checkpoint seen so far
best = res["mean_score"]
torch.save(dict(cfg=model.cfg, model=model.state_dict(), step=step, eval=res),
os.path.join(os.path.dirname(args.ckpt), "best.pt"))
with open(args.out, "a") as f:
f.write(json.dumps(res) + "\n")
print(json.dumps({k: v for k, v in res.items() if k != "best_board"}), flush=True)
if not args.watch or os.path.exists("/root/runs/WRAP_UP"):
break
time.sleep(max(10, args.min_interval - (time.time() - last_run)) if last else 10)
if __name__ == "__main__":
main()
|