| """One validation puzzle: clues, unique-solution target, stage-1 S, and |
| what the current metrics mean for a couple of cells. |
| |
| No model forward pass. The softmax rows are reconstructed to match the |
| latest eval numbers (out*~0.005, kl*~0.45, spread*~0.92) so the user can |
| see the shape, not a fake decode. |
| """ |
| import numpy as np |
|
|
| PUZ = ("/scratch/users/gatmiry/llm-reasoning-logic-puzzles/sudoku-code/" |
| "datasets/test_sudoku_puzzles.npy") |
| CAND = "/tmp/sudoku_s12/test_cand_masks.npy" |
| INST = "/tmp/sudoku_superposition" |
|
|
|
|
| def digits(m): |
| return [d for d in range(1, 10) if int(m) & (1 << (d - 1))] |
|
|
|
|
| def main(): |
| raw = np.load(PUZ, mmap_mode="r") |
| row = np.asarray(raw[0]).astype(np.int64) |
| si = int(row[0]) |
| rest = row[1:] |
| seq = np.delete(rest, np.arange(81) * 4 + 3) |
| tr = seq.reshape(81, 3) |
| masks = np.load(CAND, mmap_mode="r") |
| Sbits = np.asarray(masks[0, 0]) |
| starts = np.load(f"{INST}/test_starts.npy") |
| counts = np.load(f"{INST}/test_counts.npy") |
| A = np.load(f"{INST}/test_assignments.npy", mmap_mode="r") |
| n = int(counts[0, 0]) |
| blk = np.asarray(A[starts[0, 0]:starts[0, 0] + n]) |
|
|
| print(f"val puzzle 0: {si} clues, {81 - si} empty, {n} stored instances") |
| print("clues (input, fixed):") |
| print(" ", " ".join(f"({r},{c})={v}" for r, c, v in tr[:si][:10]), "...") |
| print("unique-solution output (NOT what val_acc scores):") |
| print(" ", " ".join(f"({r},{c})={v}" for r, c, v in tr[si:si + 8]), "...") |
|
|
| print("\nthree empty cells:") |
| print(f" {'cell':>8} {'S':<16}{'unique':>7} instance digits") |
| shown = 0 |
| cells = [] |
| for r, c, v in tr[si:]: |
| cell = int(r) * 9 + int(c) |
| S = digits(Sbits[cell]) |
| if len(S) < 2: |
| continue |
| inst = [int(x) for x in blk[:, cell]] |
| print(f" ({int(r)},{int(c)}) {str(S):<16}{int(v):>7} {inst}") |
| cells.append((cell, S, int(v), inst)) |
| shown += 1 |
| if shown == 3: |
| break |
|
|
| |
| print("\nwhat the THREE printed metrics look at (teacher-forced softmax,") |
| print("not the greedy token). A cell with S={4,5}:") |
| print(" unique-solution target would be one digit, say 4") |
| print(" training instance target is also one digit (4 or 5)") |
| print(" eval target for these metrics is the SET {4,5}") |
| print() |
| print(" typical model p at this cell right now:") |
| print(" digit 1 2 3 4 5 6 7 8 9") |
| print(" p 0 0 0 0.875 0.120 0 0 0 0.005 (tiny leak)") |
| print() |
| print(" mass* = p(4)+p(5) = 0.995") |
| print(" val_acc = out* = 1-mass* = 0.005 <-- printed val_acc") |
| print(" q = p restricted to S = (0.88, 0.12)") |
| print(" kl* = KL(uniform || q) = 0.45 <-- model prefers 4") |
| print(" leak = log(1/mass*) = 0.005") |
| print(" excess*= leak + kl* = 0.455 <-- almost all from kl*") |
| print() |
| print(" if the model were uniform over S and leaked nothing:") |
| print(" p(4)=p(5)=0.5, rest 0 -> val_acc=0 kl*=0 excess*=0") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
