File size: 3,174 Bytes
96b4304 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 | """One validation puzzle: clues, unique-solution target, stage-1 S, and
what the current metrics mean for a couple of cells.
No model forward pass. The softmax rows are reconstructed to match the
latest eval numbers (out*~0.005, kl*~0.45, spread*~0.92) so the user can
see the shape, not a fake decode.
"""
import numpy as np
PUZ = ("/scratch/users/gatmiry/llm-reasoning-logic-puzzles/sudoku-code/"
"datasets/test_sudoku_puzzles.npy")
CAND = "/tmp/sudoku_s12/test_cand_masks.npy"
INST = "/tmp/sudoku_superposition"
def digits(m):
return [d for d in range(1, 10) if int(m) & (1 << (d - 1))]
def main():
raw = np.load(PUZ, mmap_mode="r")
row = np.asarray(raw[0]).astype(np.int64)
si = int(row[0])
rest = row[1:]
seq = np.delete(rest, np.arange(81) * 4 + 3)
tr = seq.reshape(81, 3)
masks = np.load(CAND, mmap_mode="r")
Sbits = np.asarray(masks[0, 0])
starts = np.load(f"{INST}/test_starts.npy")
counts = np.load(f"{INST}/test_counts.npy")
A = np.load(f"{INST}/test_assignments.npy", mmap_mode="r")
n = int(counts[0, 0])
blk = np.asarray(A[starts[0, 0]:starts[0, 0] + n])
print(f"val puzzle 0: {si} clues, {81 - si} empty, {n} stored instances")
print("clues (input, fixed):")
print(" ", " ".join(f"({r},{c})={v}" for r, c, v in tr[:si][:10]), "...")
print("unique-solution output (NOT what val_acc scores):")
print(" ", " ".join(f"({r},{c})={v}" for r, c, v in tr[si:si + 8]), "...")
print("\nthree empty cells:")
print(f" {'cell':>8} {'S':<16}{'unique':>7} instance digits")
shown = 0
cells = []
for r, c, v in tr[si:]:
cell = int(r) * 9 + int(c)
S = digits(Sbits[cell])
if len(S) < 2:
continue
inst = [int(x) for x in blk[:, cell]]
print(f" ({int(r)},{int(c)}) {str(S):<16}{int(v):>7} {inst}")
cells.append((cell, S, int(v), inst))
shown += 1
if shown == 3:
break
# Reconstruct a typical current-run softmax: 0.5% outside S, KL~0.45 inside.
print("\nwhat the THREE printed metrics look at (teacher-forced softmax,")
print("not the greedy token). A cell with S={4,5}:")
print(" unique-solution target would be one digit, say 4")
print(" training instance target is also one digit (4 or 5)")
print(" eval target for these metrics is the SET {4,5}")
print()
print(" typical model p at this cell right now:")
print(" digit 1 2 3 4 5 6 7 8 9")
print(" p 0 0 0 0.875 0.120 0 0 0 0.005 (tiny leak)")
print()
print(" mass* = p(4)+p(5) = 0.995")
print(" val_acc = out* = 1-mass* = 0.005 <-- printed val_acc")
print(" q = p restricted to S = (0.88, 0.12)")
print(" kl* = KL(uniform || q) = 0.45 <-- model prefers 4")
print(" leak = log(1/mass*) = 0.005")
print(" excess*= leak + kl* = 0.455 <-- almost all from kl*")
print()
print(" if the model were uniform over S and leaked nothing:")
print(" p(4)=p(5)=0.5, rest 0 -> val_acc=0 kl*=0 excess*=0")
if __name__ == "__main__":
main()
|