diff --git a/_sweep_logs/master.log b/_sweep_logs/master.log new file mode 100644 index 0000000000000000000000000000000000000000..f0b5bc2a5bae06c63eac81e1c124dc1037ae14f9 --- /dev/null +++ b/_sweep_logs/master.log @@ -0,0 +1,12 @@ +===== topk_k30_s1 (k=30) ===== +===== topk_k30_s2 (k=30) ===== +===== topk_k35_s2 (k=35) ===== +===== topk_k40_s1 (k=40) ===== +===== topk_k40_s2 (k=40) ===== +===== topk_k45_s1 (k=45) ===== +===== topk_k45_s2 (k=45) ===== +===== topk_k48_s2 (k=48) ===== +===== topk_k50_s1 (k=50) ===== +===== topk_k50_s2 (k=50) ===== +===== EXTRA topk_k35_s1 (k=35) mc ===== +ALL_BARRIER_MC_DONE diff --git a/_sweep_logs/nohup.out b/_sweep_logs/nohup.out new file mode 100644 index 0000000000000000000000000000000000000000..fce75fa0bae73c9ca19091e57cc61f4cf5fe93aa --- /dev/null +++ b/_sweep_logs/nohup.out @@ -0,0 +1,90 @@ +===== topk_k30_s1 (k=30) ===== + +wrote runs/topk_k30_s1/barrier.json + +summary: MSE(α=0) = 0.05222 -> MSE(α=1) = 0.13120 + max over path = 0.13120 argmax α = 1.000 + +summary: peak=0.13200 argmax α=1.000 + endpoints: 0.05251, 0.13200 +===== topk_k30_s2 (k=30) ===== + +wrote runs/topk_k30_s2/barrier.json + +summary: MSE(α=0) = 0.05386 -> MSE(α=1) = 0.13341 + max over path = 0.13341 argmax α = 1.000 + +summary: peak=0.13334 argmax α=1.000 + endpoints: 0.05225, 0.13334 +===== topk_k35_s2 (k=35) ===== + +wrote runs/topk_k35_s2/barrier.json + +summary: MSE(α=0) = 0.04577 -> MSE(α=1) = 0.07856 + max over path = 0.07856 argmax α = 1.000 + +summary: peak=0.07992 argmax α=1.000 + endpoints: 0.04629, 0.07992 +===== topk_k40_s1 (k=40) ===== + +wrote runs/topk_k40_s1/barrier.json + +summary: MSE(α=0) = 0.03803 -> MSE(α=1) = 0.04402 + max over path = 0.05656 argmax α = 0.450 + +summary: peak=0.05201 argmax α=0.400 + endpoints: 0.03904, 0.04542 +===== topk_k40_s2 (k=40) ===== + +wrote runs/topk_k40_s2/barrier.json + +summary: MSE(α=0) = 0.03702 -> MSE(α=1) = 0.04158 + max over path = 0.05404 argmax α = 0.450 + +summary: peak=0.05049 argmax α=0.400 + endpoints: 0.03750, 0.04304 +===== topk_k45_s1 (k=45) ===== + +wrote runs/topk_k45_s1/barrier.json + +summary: MSE(α=0) = 0.03002 -> MSE(α=1) = 0.02165 + max over path = 0.06657 argmax α = 0.500 + +summary: peak=0.04251 argmax α=0.400 + endpoints: 0.02965, 0.02010 +===== topk_k45_s2 (k=45) ===== + +wrote runs/topk_k45_s2/barrier.json + +summary: MSE(α=0) = 0.03035 -> MSE(α=1) = 0.02334 + max over path = 0.06798 argmax α = 0.500 + +summary: peak=0.04420 argmax α=0.450 + endpoints: 0.02988, 0.02408 +===== topk_k48_s2 (k=48) ===== + +summary: peak=0.03999 argmax α=0.400 + endpoints: 0.02708, 0.01338 +===== topk_k50_s1 (k=50) ===== + +wrote runs/topk_k50_s1/barrier.json + +summary: MSE(α=0) = 0.02412 -> MSE(α=1) = 0.00932 + max over path = 0.05774 argmax α = 0.500 + +summary: peak=0.03846 argmax α=0.450 + endpoints: 0.02413, 0.01036 +===== topk_k50_s2 (k=50) ===== + +wrote runs/topk_k50_s2/barrier.json + +summary: MSE(α=0) = 0.02492 -> MSE(α=1) = 0.01075 + max over path = 0.06035 argmax α = 0.450 + +summary: peak=0.03823 argmax α=0.350 + endpoints: 0.02410, 0.00993 +===== EXTRA topk_k35_s1 (k=35) mc ===== + +summary: peak=0.07593 argmax α=1.000 + endpoints: 0.04496, 0.07593 +ALL_BARRIER_MC_DONE diff --git a/_sweep_logs/topk_k30_s1_barrier.log b/_sweep_logs/topk_k30_s1_barrier.log new file mode 100644 index 0000000000000000000000000000000000000000..982e9ecc38d2c4c645101f82850dac9392414f91 --- /dev/null +++ b/_sweep_logs/topk_k30_s1_barrier.log @@ -0,0 +1,30 @@ + +barrier eval: run=runs/topk_k30_s1 k=30 n=5000 alphas=21 + +Hungarian alignment: mean cos = 0.6010 (0.6010 per scipy) + α=0.000 MSE=0.05222 + α=0.050 MSE=0.05257 + α=0.100 MSE=0.05410 + α=0.150 MSE=0.05670 + α=0.200 MSE=0.06035 + α=0.250 MSE=0.06481 + α=0.300 MSE=0.06998 + α=0.350 MSE=0.07554 + α=0.400 MSE=0.08108 + α=0.450 MSE=0.08637 + α=0.500 MSE=0.09153 + α=0.550 MSE=0.09655 + α=0.600 MSE=0.10156 + α=0.650 MSE=0.10641 + α=0.700 MSE=0.11103 + α=0.750 MSE=0.11534 + α=0.800 MSE=0.11899 + α=0.850 MSE=0.12208 + α=0.900 MSE=0.12484 + α=0.950 MSE=0.12781 + α=1.000 MSE=0.13120 + +wrote runs/topk_k30_s1/barrier.json + +summary: MSE(α=0) = 0.05222 -> MSE(α=1) = 0.13120 + max over path = 0.13120 argmax α = 1.000 diff --git a/_sweep_logs/topk_k30_s2_barrier.log b/_sweep_logs/topk_k30_s2_barrier.log new file mode 100644 index 0000000000000000000000000000000000000000..1eb5d7262724fcf1f2c31da99ff8552211abf10a --- /dev/null +++ b/_sweep_logs/topk_k30_s2_barrier.log @@ -0,0 +1,30 @@ + +barrier eval: run=runs/topk_k30_s2 k=30 n=5000 alphas=21 + +Hungarian alignment: mean cos = 0.5997 (0.5997 per scipy) + α=0.000 MSE=0.05386 + α=0.050 MSE=0.05409 + α=0.100 MSE=0.05553 + α=0.150 MSE=0.05806 + α=0.200 MSE=0.06159 + α=0.250 MSE=0.06596 + α=0.300 MSE=0.07100 + α=0.350 MSE=0.07646 + α=0.400 MSE=0.08206 + α=0.450 MSE=0.08748 + α=0.500 MSE=0.09287 + α=0.550 MSE=0.09813 + α=0.600 MSE=0.10334 + α=0.650 MSE=0.10837 + α=0.700 MSE=0.11316 + α=0.750 MSE=0.11754 + α=0.800 MSE=0.12121 + α=0.850 MSE=0.12434 + α=0.900 MSE=0.12713 + α=0.950 MSE=0.13001 + α=1.000 MSE=0.13341 + +wrote runs/topk_k30_s2/barrier.json + +summary: MSE(α=0) = 0.05386 -> MSE(α=1) = 0.13341 + max over path = 0.13341 argmax α = 1.000 diff --git a/_sweep_logs/topk_k40_s1_mc.log b/_sweep_logs/topk_k40_s1_mc.log new file mode 100644 index 0000000000000000000000000000000000000000..f44b95b1f135b3cf23dfb53f42da409b31a4fac1 --- /dev/null +++ b/_sweep_logs/topk_k40_s1_mc.log @@ -0,0 +1,43 @@ + +mode connectivity: run=runs/topk_k40_s1 k=40 train α=[0.2, 0.4, 0.5, 0.6, 0.8] + +optimizing midpoint dictionary (80 steps, batch 128)... + step 1 meanMSE=0.05600 [0.20:0.0436 0.40:0.0578 0.50:0.0611 0.60:0.0611 0.80:0.0564] + step 10 meanMSE=0.05563 [0.20:0.0486 0.40:0.0597 0.50:0.0603 0.60:0.0581 0.80:0.0515] + step 20 meanMSE=0.04781 [0.20:0.0420 0.40:0.0526 0.50:0.0525 0.60:0.0498 0.80:0.0423] + step 30 meanMSE=0.05105 [0.20:0.0463 0.40:0.0557 0.50:0.0553 0.60:0.0523 0.80:0.0456] + step 40 meanMSE=0.05143 [0.20:0.0469 0.40:0.0551 0.50:0.0550 0.60:0.0528 0.80:0.0473] + step 50 meanMSE=0.04723 [0.20:0.0418 0.40:0.0516 0.50:0.0517 0.60:0.0494 0.80:0.0417] + step 60 meanMSE=0.05595 [0.20:0.0500 0.40:0.0590 0.50:0.0592 0.60:0.0571 0.80:0.0545] + step 70 meanMSE=0.04839 [0.20:0.0472 0.40:0.0533 0.50:0.0521 0.60:0.0487 0.80:0.0407] + step 80 meanMSE=0.04619 [0.20:0.0445 0.40:0.0507 0.50:0.0499 0.60:0.0465 0.80:0.0395] + +optimised in 13.0 sec + +evaluating optimised curve (21 αs × 5000 samples) + α=0.000 MSE=0.03904 + α=0.050 MSE=0.03948 + α=0.100 MSE=0.04142 + α=0.150 MSE=0.04389 + α=0.200 MSE=0.04638 + α=0.250 MSE=0.04857 + α=0.300 MSE=0.05028 + α=0.350 MSE=0.05146 + α=0.400 MSE=0.05201 + α=0.450 MSE=0.05191 + α=0.500 MSE=0.05122 + α=0.550 MSE=0.04995 + α=0.600 MSE=0.04826 + α=0.650 MSE=0.04631 + α=0.700 MSE=0.04447 + α=0.750 MSE=0.04308 + α=0.800 MSE=0.04241 + α=0.850 MSE=0.04253 + α=0.900 MSE=0.04318 + α=0.950 MSE=0.04408 + α=1.000 MSE=0.04542 + +wrote runs/topk_k40_s1/mode_connectivity.json + +summary: peak=0.05201 argmax α=0.400 + endpoints: 0.03904, 0.04542 diff --git a/_sweep_logs/topk_k40_s2_barrier.log b/_sweep_logs/topk_k40_s2_barrier.log new file mode 100644 index 0000000000000000000000000000000000000000..f2548fee008f53e546a243577c6af3c5e621bf52 --- /dev/null +++ b/_sweep_logs/topk_k40_s2_barrier.log @@ -0,0 +1,30 @@ + +barrier eval: run=runs/topk_k40_s2 k=40 n=5000 alphas=21 + +Hungarian alignment: mean cos = 0.6576 (0.6576 per scipy) + α=0.000 MSE=0.03702 + α=0.050 MSE=0.03730 + α=0.100 MSE=0.03833 + α=0.150 MSE=0.04001 + α=0.200 MSE=0.04229 + α=0.250 MSE=0.04503 + α=0.300 MSE=0.04802 + α=0.350 MSE=0.05084 + α=0.400 MSE=0.05300 + α=0.450 MSE=0.05404 + α=0.500 MSE=0.05397 + α=0.550 MSE=0.05314 + α=0.600 MSE=0.05162 + α=0.650 MSE=0.04985 + α=0.700 MSE=0.04784 + α=0.750 MSE=0.04588 + α=0.800 MSE=0.04399 + α=0.850 MSE=0.04239 + α=0.900 MSE=0.04129 + α=0.950 MSE=0.04095 + α=1.000 MSE=0.04158 + +wrote runs/topk_k40_s2/barrier.json + +summary: MSE(α=0) = 0.03702 -> MSE(α=1) = 0.04158 + max over path = 0.05404 argmax α = 0.450 diff --git a/_sweep_logs/topk_k45_s1_barrier.log b/_sweep_logs/topk_k45_s1_barrier.log new file mode 100644 index 0000000000000000000000000000000000000000..b0bafece7a0c4990d43cf79085d6a1a467659129 --- /dev/null +++ b/_sweep_logs/topk_k45_s1_barrier.log @@ -0,0 +1,30 @@ + +barrier eval: run=runs/topk_k45_s1 k=45 n=5000 alphas=21 + +Hungarian alignment: mean cos = 0.6868 (0.6868 per scipy) + α=0.000 MSE=0.03002 + α=0.050 MSE=0.03061 + α=0.100 MSE=0.03255 + α=0.150 MSE=0.03566 + α=0.200 MSE=0.03990 + α=0.250 MSE=0.04514 + α=0.300 MSE=0.05113 + α=0.350 MSE=0.05728 + α=0.400 MSE=0.06258 + α=0.450 MSE=0.06588 + α=0.500 MSE=0.06657 + α=0.550 MSE=0.06490 + α=0.600 MSE=0.06119 + α=0.650 MSE=0.05584 + α=0.700 MSE=0.04892 + α=0.750 MSE=0.04164 + α=0.800 MSE=0.03480 + α=0.850 MSE=0.02899 + α=0.900 MSE=0.02470 + α=0.950 MSE=0.02214 + α=1.000 MSE=0.02165 + +wrote runs/topk_k45_s1/barrier.json + +summary: MSE(α=0) = 0.03002 -> MSE(α=1) = 0.02165 + max over path = 0.06657 argmax α = 0.500 diff --git a/_sweep_logs/topk_k45_s1_mc.log b/_sweep_logs/topk_k45_s1_mc.log new file mode 100644 index 0000000000000000000000000000000000000000..f64d3780be9bf708b97dceff63339c19395d7f64 --- /dev/null +++ b/_sweep_logs/topk_k45_s1_mc.log @@ -0,0 +1,43 @@ + +mode connectivity: run=runs/topk_k45_s1 k=45 train α=[0.2, 0.4, 0.5, 0.6, 0.8] + +optimizing midpoint dictionary (80 steps, batch 128)... + step 1 meanMSE=0.05412 [0.20:0.0382 0.40:0.0623 0.50:0.0674 0.60:0.0636 0.80:0.0391] + step 10 meanMSE=0.04365 [0.20:0.0362 0.40:0.0474 0.50:0.0489 0.60:0.0481 0.80:0.0377] + step 20 meanMSE=0.04038 [0.20:0.0373 0.40:0.0472 0.50:0.0464 0.60:0.0417 0.80:0.0293] + step 30 meanMSE=0.04035 [0.20:0.0373 0.40:0.0464 0.50:0.0453 0.60:0.0414 0.80:0.0314] + step 40 meanMSE=0.04231 [0.20:0.0401 0.40:0.0482 0.50:0.0471 0.60:0.0430 0.80:0.0331] + step 50 meanMSE=0.04177 [0.20:0.0402 0.40:0.0487 0.50:0.0478 0.60:0.0429 0.80:0.0292] + step 60 meanMSE=0.04048 [0.20:0.0378 0.40:0.0461 0.50:0.0456 0.60:0.0422 0.80:0.0306] + step 70 meanMSE=0.03406 [0.20:0.0327 0.40:0.0394 0.50:0.0392 0.60:0.0357 0.80:0.0234] + step 80 meanMSE=0.04230 [0.20:0.0406 0.40:0.0468 0.50:0.0462 0.60:0.0435 0.80:0.0344] + +optimised in 15.7 sec + +evaluating optimised curve (21 αs × 5000 samples) + α=0.000 MSE=0.02965 + α=0.050 MSE=0.03000 + α=0.100 MSE=0.03170 + α=0.150 MSE=0.03396 + α=0.200 MSE=0.03634 + α=0.250 MSE=0.03857 + α=0.300 MSE=0.04046 + α=0.350 MSE=0.04184 + α=0.400 MSE=0.04251 + α=0.450 MSE=0.04244 + α=0.500 MSE=0.04160 + α=0.550 MSE=0.04011 + α=0.600 MSE=0.03800 + α=0.650 MSE=0.03541 + α=0.700 MSE=0.03256 + α=0.750 MSE=0.02964 + α=0.800 MSE=0.02683 + α=0.850 MSE=0.02429 + α=0.900 MSE=0.02207 + α=0.950 MSE=0.02045 + α=1.000 MSE=0.02010 + +wrote runs/topk_k45_s1/mode_connectivity.json + +summary: peak=0.04251 argmax α=0.400 + endpoints: 0.02965, 0.02010 diff --git a/_sweep_logs/topk_k45_s2_mc.log b/_sweep_logs/topk_k45_s2_mc.log new file mode 100644 index 0000000000000000000000000000000000000000..7790ba5d13f355915ce98c417253be88d38aac71 --- /dev/null +++ b/_sweep_logs/topk_k45_s2_mc.log @@ -0,0 +1,43 @@ + +mode connectivity: run=runs/topk_k45_s2 k=45 train α=[0.2, 0.4, 0.5, 0.6, 0.8] + +optimizing midpoint dictionary (80 steps, batch 128)... + step 1 meanMSE=0.05019 [0.20:0.0415 0.40:0.0622 0.50:0.0635 0.60:0.0562 0.80:0.0275] + step 10 meanMSE=0.03718 [0.20:0.0361 0.40:0.0443 0.50:0.0429 0.60:0.0384 0.80:0.0242] + step 20 meanMSE=0.04184 [0.20:0.0359 0.40:0.0470 0.50:0.0472 0.60:0.0445 0.80:0.0345] + step 30 meanMSE=0.04091 [0.20:0.0369 0.40:0.0466 0.50:0.0463 0.60:0.0427 0.80:0.0321] + step 40 meanMSE=0.03408 [0.20:0.0324 0.40:0.0405 0.50:0.0396 0.60:0.0351 0.80:0.0228] + step 50 meanMSE=0.03986 [0.20:0.0386 0.40:0.0465 0.50:0.0452 0.60:0.0409 0.80:0.0281] + step 60 meanMSE=0.03756 [0.20:0.0387 0.40:0.0441 0.50:0.0423 0.60:0.0376 0.80:0.0253] + step 70 meanMSE=0.03755 [0.20:0.0356 0.40:0.0425 0.50:0.0417 0.60:0.0383 0.80:0.0297] + step 80 meanMSE=0.03776 [0.20:0.0385 0.40:0.0435 0.50:0.0423 0.60:0.0383 0.80:0.0262] + +optimised in 15.6 sec + +evaluating optimised curve (21 αs × 5000 samples) + α=0.000 MSE=0.02988 + α=0.050 MSE=0.03029 + α=0.100 MSE=0.03216 + α=0.150 MSE=0.03468 + α=0.200 MSE=0.03731 + α=0.250 MSE=0.03977 + α=0.300 MSE=0.04187 + α=0.350 MSE=0.04334 + α=0.400 MSE=0.04415 + α=0.450 MSE=0.04420 + α=0.500 MSE=0.04354 + α=0.550 MSE=0.04225 + α=0.600 MSE=0.04033 + α=0.650 MSE=0.03791 + α=0.700 MSE=0.03530 + α=0.750 MSE=0.03261 + α=0.800 MSE=0.03006 + α=0.850 MSE=0.02778 + α=0.900 MSE=0.02578 + α=0.950 MSE=0.02422 + α=1.000 MSE=0.02408 + +wrote runs/topk_k45_s2/mode_connectivity.json + +summary: peak=0.04420 argmax α=0.450 + endpoints: 0.02988, 0.02408 diff --git a/_sweep_logs/topk_k48_s2_mc.log b/_sweep_logs/topk_k48_s2_mc.log new file mode 100644 index 0000000000000000000000000000000000000000..72626694766d00b7f65d74d784ea582dccba4c58 --- /dev/null +++ b/_sweep_logs/topk_k48_s2_mc.log @@ -0,0 +1,43 @@ + +mode connectivity: run=runs/topk_k48_s2 k=48 train α=[0.2, 0.4, 0.5, 0.6, 0.8] + +optimizing midpoint dictionary (80 steps, batch 128)... + step 1 meanMSE=0.04885 [0.20:0.0373 0.40:0.0593 0.50:0.0621 0.60:0.0562 0.80:0.0293] + step 10 meanMSE=0.03331 [0.20:0.0332 0.40:0.0413 0.50:0.0398 0.60:0.0346 0.80:0.0176] + step 20 meanMSE=0.02877 [0.20:0.0288 0.40:0.0365 0.50:0.0346 0.60:0.0296 0.80:0.0144] + step 30 meanMSE=0.03208 [0.20:0.0326 0.40:0.0393 0.50:0.0375 0.60:0.0328 0.80:0.0182] + step 40 meanMSE=0.03851 [0.20:0.0376 0.40:0.0461 0.50:0.0448 0.60:0.0395 0.80:0.0246] + step 50 meanMSE=0.03263 [0.20:0.0313 0.40:0.0386 0.50:0.0377 0.60:0.0339 0.80:0.0216] + step 60 meanMSE=0.03791 [0.20:0.0366 0.40:0.0436 0.50:0.0430 0.60:0.0394 0.80:0.0270] + step 70 meanMSE=0.03371 [0.20:0.0348 0.40:0.0404 0.50:0.0388 0.60:0.0343 0.80:0.0202] + step 80 meanMSE=0.03407 [0.20:0.0333 0.40:0.0400 0.50:0.0393 0.60:0.0356 0.80:0.0222] + +optimised in 16.9 sec + +evaluating optimised curve (21 αs × 5000 samples) + α=0.000 MSE=0.02708 + α=0.050 MSE=0.02740 + α=0.100 MSE=0.02909 + α=0.150 MSE=0.03136 + α=0.200 MSE=0.03374 + α=0.250 MSE=0.03599 + α=0.300 MSE=0.03788 + α=0.350 MSE=0.03925 + α=0.400 MSE=0.03999 + α=0.450 MSE=0.03998 + α=0.500 MSE=0.03920 + α=0.550 MSE=0.03772 + α=0.600 MSE=0.03554 + α=0.650 MSE=0.03279 + α=0.700 MSE=0.02966 + α=0.750 MSE=0.02631 + α=0.800 MSE=0.02289 + α=0.850 MSE=0.01957 + α=0.900 MSE=0.01650 + α=0.950 MSE=0.01408 + α=1.000 MSE=0.01338 + +wrote runs/topk_k48_s2/mode_connectivity.json + +summary: peak=0.03999 argmax α=0.400 + endpoints: 0.02708, 0.01338 diff --git a/_sweep_logs/topk_k50_s1_barrier.log b/_sweep_logs/topk_k50_s1_barrier.log new file mode 100644 index 0000000000000000000000000000000000000000..7fba1c4aab7e87400275a5893ef2358ccac77032 --- /dev/null +++ b/_sweep_logs/topk_k50_s1_barrier.log @@ -0,0 +1,30 @@ + +barrier eval: run=runs/topk_k50_s1 k=50 n=5000 alphas=21 + +Hungarian alignment: mean cos = 0.7114 (0.7114 per scipy) + α=0.000 MSE=0.02412 + α=0.050 MSE=0.02465 + α=0.100 MSE=0.02638 + α=0.150 MSE=0.02917 + α=0.200 MSE=0.03299 + α=0.250 MSE=0.03777 + α=0.300 MSE=0.04328 + α=0.350 MSE=0.04895 + α=0.400 MSE=0.05391 + α=0.450 MSE=0.05709 + α=0.500 MSE=0.05774 + α=0.550 MSE=0.05586 + α=0.600 MSE=0.05166 + α=0.650 MSE=0.04569 + α=0.700 MSE=0.03836 + α=0.750 MSE=0.03068 + α=0.800 MSE=0.02352 + α=0.850 MSE=0.01745 + α=0.900 MSE=0.01289 + α=0.950 MSE=0.01013 + α=1.000 MSE=0.00932 + +wrote runs/topk_k50_s1/barrier.json + +summary: MSE(α=0) = 0.02412 -> MSE(α=1) = 0.00932 + max over path = 0.05774 argmax α = 0.500 diff --git a/_sweep_logs/topk_k50_s1_mc.log b/_sweep_logs/topk_k50_s1_mc.log new file mode 100644 index 0000000000000000000000000000000000000000..374448fef3ffbc121969e6951eaa9fff96b49c3e --- /dev/null +++ b/_sweep_logs/topk_k50_s1_mc.log @@ -0,0 +1,43 @@ + +mode connectivity: run=runs/topk_k50_s1 k=50 train α=[0.2, 0.4, 0.5, 0.6, 0.8] + +optimizing midpoint dictionary (80 steps, batch 128)... + step 1 meanMSE=0.04677 [0.20:0.0357 0.40:0.0571 0.50:0.0609 0.60:0.0543 0.80:0.0259] + step 10 meanMSE=0.02857 [0.20:0.0274 0.40:0.0351 0.50:0.0343 0.60:0.0301 0.80:0.0160] + step 20 meanMSE=0.02857 [0.20:0.0291 0.40:0.0366 0.50:0.0344 0.60:0.0292 0.80:0.0135] + step 30 meanMSE=0.03326 [0.20:0.0298 0.40:0.0391 0.50:0.0389 0.60:0.0357 0.80:0.0229] + step 40 meanMSE=0.03114 [0.20:0.0318 0.40:0.0389 0.50:0.0369 0.60:0.0318 0.80:0.0164] + step 50 meanMSE=0.03590 [0.20:0.0338 0.40:0.0428 0.50:0.0420 0.60:0.0376 0.80:0.0233] + step 60 meanMSE=0.03094 [0.20:0.0300 0.40:0.0372 0.50:0.0365 0.60:0.0326 0.80:0.0184] + step 70 meanMSE=0.03008 [0.20:0.0301 0.40:0.0367 0.50:0.0358 0.60:0.0316 0.80:0.0162] + step 80 meanMSE=0.02948 [0.20:0.0270 0.40:0.0348 0.50:0.0349 0.60:0.0317 0.80:0.0190] + +optimised in 18.1 sec + +evaluating optimised curve (21 αs × 5000 samples) + α=0.000 MSE=0.02413 + α=0.050 MSE=0.02446 + α=0.100 MSE=0.02617 + α=0.150 MSE=0.02854 + α=0.200 MSE=0.03113 + α=0.250 MSE=0.03361 + α=0.300 MSE=0.03572 + α=0.350 MSE=0.03732 + α=0.400 MSE=0.03828 + α=0.450 MSE=0.03846 + α=0.500 MSE=0.03783 + α=0.550 MSE=0.03636 + α=0.600 MSE=0.03418 + α=0.650 MSE=0.03141 + α=0.700 MSE=0.02818 + α=0.750 MSE=0.02462 + α=0.800 MSE=0.02094 + α=0.850 MSE=0.01723 + α=0.900 MSE=0.01385 + α=0.950 MSE=0.01122 + α=1.000 MSE=0.01036 + +wrote runs/topk_k50_s1/mode_connectivity.json + +summary: peak=0.03846 argmax α=0.450 + endpoints: 0.02413, 0.01036 diff --git a/_sweep_logs/topk_k50_s2_barrier.log b/_sweep_logs/topk_k50_s2_barrier.log new file mode 100644 index 0000000000000000000000000000000000000000..2d0a0c4265dc6a326f9ddf6c63326372e3157247 --- /dev/null +++ b/_sweep_logs/topk_k50_s2_barrier.log @@ -0,0 +1,30 @@ + +barrier eval: run=runs/topk_k50_s2 k=50 n=5000 alphas=21 + +Hungarian alignment: mean cos = 0.7091 (0.7091 per scipy) + α=0.000 MSE=0.02492 + α=0.050 MSE=0.02552 + α=0.100 MSE=0.02761 + α=0.150 MSE=0.03107 + α=0.200 MSE=0.03573 + α=0.250 MSE=0.04131 + α=0.300 MSE=0.04735 + α=0.350 MSE=0.05317 + α=0.400 MSE=0.05783 + α=0.450 MSE=0.06035 + α=0.500 MSE=0.06025 + α=0.550 MSE=0.05780 + α=0.600 MSE=0.05331 + α=0.650 MSE=0.04725 + α=0.700 MSE=0.03984 + α=0.750 MSE=0.03205 + α=0.800 MSE=0.02483 + α=0.850 MSE=0.01874 + α=0.900 MSE=0.01422 + α=0.950 MSE=0.01150 + α=1.000 MSE=0.01075 + +wrote runs/topk_k50_s2/barrier.json + +summary: MSE(α=0) = 0.02492 -> MSE(α=1) = 0.01075 + max over path = 0.06035 argmax α = 0.450 diff --git a/altmin_perturbed_omp_k48_s43_c0.30/results.json b/altmin_perturbed_omp_k48_s43_c0.30/results.json new file mode 100644 index 0000000000000000000000000000000000000000..8242f18a21a33796253a367003624fa6c98481ec --- /dev/null +++ b/altmin_perturbed_omp_k48_s43_c0.30/results.json @@ -0,0 +1,23 @@ +{ + "init": "perturbed", + "coding": "omp", + "k": 48, + "n_samples": 10000000, + "lr": 0.0003, + "seed": 43, + "eval": { + "true_l0": 34.493215, + "sae_l0": 38.94516, + "dead_latents": 0, + "shrinkage": 0.9630266931152344, + "explained_variance": 0.9091709596347162, + "mcc": 0.9172072410583496, + "uniqueness": 0.9921875, + "classification/precision": 0.6697086691856384, + "classification/recall": 0.6960408687591553, + "classification/f1_score": 0.6599588394165039, + "classification/accuracy": 0.9984580278396606 + }, + "dec_fidelity_final": 0.9184098839759827, + "seconds": 3053.9301381111145 +} \ No newline at end of file diff --git a/altmin_perturbed_omp_k48_s43_c0.40/results.json b/altmin_perturbed_omp_k48_s43_c0.40/results.json new file mode 100644 index 0000000000000000000000000000000000000000..c671961d45ece23fd6eb8d2b33f8f062ca1163c3 --- /dev/null +++ b/altmin_perturbed_omp_k48_s43_c0.40/results.json @@ -0,0 +1,23 @@ +{ + "init": "perturbed", + "coding": "omp", + "k": 48, + "n_samples": 10000000, + "lr": 0.0003, + "seed": 43, + "eval": { + "true_l0": 34.493215, + "sae_l0": 39.43884, + "dead_latents": 0, + "shrinkage": 0.9721412866210938, + "explained_variance": 0.9291685954011948, + "mcc": 0.9540121555328369, + "uniqueness": 0.99951171875, + "classification/precision": 0.7348754405975342, + "classification/recall": 0.750920832157135, + "classification/f1_score": 0.7241045236587524, + "classification/accuracy": 0.9988424777984619 + }, + "dec_fidelity_final": 0.9541122913360596, + "seconds": 3065.881599664688 +} \ No newline at end of file diff --git a/basin_radius/results_seed42.json b/basin_radius/results_seed42.json new file mode 100644 index 0000000000000000000000000000000000000000..6739ac1883ca4dd540b25fbef87bd27cce2ba7e7 --- /dev/null +++ b/basin_radius/results_seed42.json @@ -0,0 +1,104 @@ +{ + "k": 48, + "n_steps": 500, + "n_samples": 2048000, + "seed": 42, + "results": [ + { + "target_cos": 0.99, + "fid_init": 0.9900000095367432, + "fid_final": 0.9861536026000977, + "delta": -0.003846406936645508, + "converging": false, + "seconds": 448.9438157081604 + }, + { + "target_cos": 0.97, + "fid_init": 0.9700000286102295, + "fid_final": 0.9848216772079468, + "delta": 0.014821648597717285, + "converging": true, + "seconds": 452.4573805332184 + }, + { + "target_cos": 0.95, + "fid_init": 0.9500000476837158, + "fid_final": 0.9826305508613586, + "delta": 0.03263050317764282, + "converging": true, + "seconds": 453.744690656662 + }, + { + "target_cos": 0.9, + "fid_init": 0.9000000953674316, + "fid_final": 0.9756519198417664, + "delta": 0.07565182447433472, + "converging": true, + "seconds": 455.57620120048523 + }, + { + "target_cos": 0.85, + "fid_init": 0.8500000238418579, + "fid_final": 0.9676114320755005, + "delta": 0.11761140823364258, + "converging": true, + "seconds": 456.31761837005615 + }, + { + "target_cos": 0.8, + "fid_init": 0.8000000715255737, + "fid_final": 0.9589360356330872, + "delta": 0.15893596410751343, + "converging": true, + "seconds": 455.580677986145 + }, + { + "target_cos": 0.7, + "fid_init": 0.7118338346481323, + "fid_final": 0.8864246010780334, + "delta": 0.17459076642990112, + "converging": true, + "seconds": 455.87197256088257 + }, + { + "target_cos": 0.6, + "fid_init": 0.7552973031997681, + "fid_final": 0.866296648979187, + "delta": 0.11099934577941895, + "converging": true, + "seconds": 455.4593770503998 + }, + { + "target_cos": 0.5, + "fid_init": 0.7880702614784241, + "fid_final": 0.8552908897399902, + "delta": 0.06722062826156616, + "converging": true, + "seconds": 455.7541358470917 + }, + { + "target_cos": 0.3, + "fid_init": 0.8184905052185059, + "fid_final": 0.8534693717956543, + "delta": 0.03497886657714844, + "converging": true, + "seconds": 456.12142276763916 + }, + { + "target_cos": 0.2, + "fid_init": 0.8192972540855408, + "fid_final": 0.8383699655532837, + "delta": 0.01907271146774292, + "converging": true, + "seconds": 455.7897198200226 + }, + { + "target_cos": 0.1, + "fid_init": 0.8211536407470703, + "fid_final": 0.8282573223114014, + "delta": 0.007103681564331055, + "converging": true, + "seconds": 455.4574017524719 + } + ] +} \ No newline at end of file diff --git a/basin_radius/results_seed43.json b/basin_radius/results_seed43.json new file mode 100644 index 0000000000000000000000000000000000000000..b418772f7e3002907157b2288b61561fb24f0dad --- /dev/null +++ b/basin_radius/results_seed43.json @@ -0,0 +1,104 @@ +{ + "k": 48, + "n_steps": 500, + "n_samples": 2048000, + "seed": 43, + "results": [ + { + "target_cos": 0.99, + "fid_init": 0.9900000095367432, + "fid_final": 0.9860057830810547, + "delta": -0.0039942264556884766, + "converging": false, + "seconds": 448.4635183811188 + }, + { + "target_cos": 0.97, + "fid_init": 0.9700000286102295, + "fid_final": 0.9846746921539307, + "delta": 0.014674663543701172, + "converging": true, + "seconds": 447.87573170661926 + }, + { + "target_cos": 0.95, + "fid_init": 0.9500000476837158, + "fid_final": 0.9829118251800537, + "delta": 0.03291177749633789, + "converging": true, + "seconds": 448.0157024860382 + }, + { + "target_cos": 0.9, + "fid_init": 0.9000000953674316, + "fid_final": 0.9769527316093445, + "delta": 0.07695263624191284, + "converging": true, + "seconds": 448.6988363265991 + }, + { + "target_cos": 0.85, + "fid_init": 0.8500000238418579, + "fid_final": 0.969549298286438, + "delta": 0.11954927444458008, + "converging": true, + "seconds": 448.3461797237396 + }, + { + "target_cos": 0.8, + "fid_init": 0.8000000715255737, + "fid_final": 0.9612268209457397, + "delta": 0.16122674942016602, + "converging": true, + "seconds": 448.9183044433594 + }, + { + "target_cos": 0.7, + "fid_init": 0.7000000476837158, + "fid_final": 0.9416828155517578, + "delta": 0.241682767868042, + "converging": true, + "seconds": 449.0006408691406 + }, + { + "target_cos": 0.6, + "fid_init": 0.6000000238418579, + "fid_final": 0.9099850654602051, + "delta": 0.30998504161834717, + "converging": true, + "seconds": 449.1651108264923 + }, + { + "target_cos": 0.5, + "fid_init": 0.5, + "fid_final": 0.8409985303878784, + "delta": 0.3409985303878784, + "converging": true, + "seconds": 450.0478971004486 + }, + { + "target_cos": 0.3, + "fid_init": 0.30000001192092896, + "fid_final": 0.41315793991088867, + "delta": 0.11315792798995972, + "converging": true, + "seconds": 448.98216104507446 + }, + { + "target_cos": 0.2, + "fid_init": 0.20000237226486206, + "fid_final": 0.2386893630027771, + "delta": 0.03868699073791504, + "converging": true, + "seconds": 449.53068566322327 + }, + { + "target_cos": 0.1, + "fid_init": 0.14860697090625763, + "fid_final": 0.21956323087215424, + "delta": 0.0709562599658966, + "converging": true, + "seconds": 450.1033709049225 + } + ] +} \ No newline at end of file diff --git a/basin_radius/results_seed44.json b/basin_radius/results_seed44.json new file mode 100644 index 0000000000000000000000000000000000000000..80e8d8af9fc8fb34cecd7c63efcd97d0a5b8a482 --- /dev/null +++ b/basin_radius/results_seed44.json @@ -0,0 +1,104 @@ +{ + "k": 48, + "n_steps": 500, + "n_samples": 2048000, + "seed": 44, + "results": [ + { + "target_cos": 0.99, + "fid_init": 0.9900000095367432, + "fid_final": 0.9861334562301636, + "delta": -0.00386655330657959, + "converging": false, + "seconds": 458.3089339733124 + }, + { + "target_cos": 0.97, + "fid_init": 0.9700000286102295, + "fid_final": 0.9848356246948242, + "delta": 0.014835596084594727, + "converging": true, + "seconds": 461.82796478271484 + }, + { + "target_cos": 0.95, + "fid_init": 0.9500000476837158, + "fid_final": 0.983006477355957, + "delta": 0.03300642967224121, + "converging": true, + "seconds": 461.8896379470825 + }, + { + "target_cos": 0.9, + "fid_init": 0.9000000953674316, + "fid_final": 0.9769294261932373, + "delta": 0.07692933082580566, + "converging": true, + "seconds": 460.7876327037811 + }, + { + "target_cos": 0.85, + "fid_init": 0.8500000238418579, + "fid_final": 0.9694505333900452, + "delta": 0.11945050954818726, + "converging": true, + "seconds": 461.25571632385254 + }, + { + "target_cos": 0.8, + "fid_init": 0.8000000715255737, + "fid_final": 0.9610999822616577, + "delta": 0.16109991073608398, + "converging": true, + "seconds": 465.52943754196167 + }, + { + "target_cos": 0.7, + "fid_init": 0.7000000476837158, + "fid_final": 0.9415371417999268, + "delta": 0.24153709411621094, + "converging": true, + "seconds": 461.8532736301422 + }, + { + "target_cos": 0.6, + "fid_init": 0.6000000238418579, + "fid_final": 0.9099147319793701, + "delta": 0.3099147081375122, + "converging": true, + "seconds": 463.27074575424194 + }, + { + "target_cos": 0.5, + "fid_init": 0.5, + "fid_final": 0.8409147262573242, + "delta": 0.3409147262573242, + "converging": true, + "seconds": 464.4487407207489 + }, + { + "target_cos": 0.3, + "fid_init": 0.30000001192092896, + "fid_final": 0.4143797755241394, + "delta": 0.11437976360321045, + "converging": true, + "seconds": 463.4620110988617 + }, + { + "target_cos": 0.2, + "fid_init": 0.20000238716602325, + "fid_final": 0.24059367179870605, + "delta": 0.0405912846326828, + "converging": true, + "seconds": 457.25708651542664 + }, + { + "target_cos": 0.1, + "fid_init": 0.14846540987491608, + "fid_final": 0.22045353055000305, + "delta": 0.07198812067508698, + "converging": true, + "seconds": 459.9436514377594 + } + ] +} \ No newline at end of file diff --git a/frozen_dec_k48_s1/results.json b/frozen_dec_k48_s1/results.json new file mode 100644 index 0000000000000000000000000000000000000000..aec3d31030ea31052b676219589a1067c3cc22e7 --- /dev/null +++ b/frozen_dec_k48_s1/results.json @@ -0,0 +1,26 @@ +{ + "k": 48, + "n_samples": 15000000, + "lr": 0.0003, + "seed": 1, + "support_overlap_init": { + "jaccard": 0.4592665173012477, + "recall": 0.6161499023437506 + }, + "support_overlap_final": { + "jaccard": 0.5475213736319267, + "recall": 0.6813151041666667 + }, + "dec_fidelity_init": 1.0, + "dec_fidelity_final": 1.0, + "eval": { + "true_l0": 34.46586, + "sae_l0": 47.98879, + "dead_latents": 0, + "shrinkage": 0.9727815844726563, + "explained_variance": 0.9287085583622611, + "mcc": 1.0, + "uniqueness": 1.0 + }, + "seconds": 160.0245246887207 +} \ No newline at end of file diff --git a/oracle_init_smoke/cfg.json b/oracle_init_smoke/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..eb699e0232a81e63365b5fa690a5ddd0b6911ad4 --- /dev/null +++ b/oracle_init_smoke/cfg.json @@ -0,0 +1 @@ +{"normalize_activations": "none", "dtype": "float32", "device": "cuda", "d_in": 768, "rescale_acts_by_decoder_norm": true, "apply_b_dec_to_input": true, "d_sae": 16384, "k": 48, "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "reshape_activations": "none", "architecture": "topk"} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/114688/activation_scaler.json b/oracle_init_smoke/ckpts/114688/activation_scaler.json new file mode 100644 index 0000000000000000000000000000000000000000..71489f84b6af6cbeb92d8c3485dcc2dd40e4c375 --- /dev/null +++ b/oracle_init_smoke/ckpts/114688/activation_scaler.json @@ -0,0 +1 @@ +{"scaling_factor": 0.9888814189820523} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/114688/cfg.json b/oracle_init_smoke/ckpts/114688/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..2ae82a94121a05017b7424f4caca7141c9a59c56 --- /dev/null +++ b/oracle_init_smoke/ckpts/114688/cfg.json @@ -0,0 +1 @@ +{"d_in": 768, "d_sae": 16384, "dtype": "float32", "device": "cuda", "apply_b_dec_to_input": true, "normalize_activations": "expected_average_only_in", "reshape_activations": "none", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "decoder_init_norm": 0.1, "k": 48, "use_sparse_activations": false, "aux_loss_coefficient": 1.0, "rescale_acts_by_decoder_norm": true, "architecture": "topk"} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/114688/runner_config.json b/oracle_init_smoke/ckpts/114688/runner_config.json new file mode 100644 index 0000000000000000000000000000000000000000..36b25139fd4b0bc732412b5c1df03af29e4fa5f9 --- /dev/null +++ b/oracle_init_smoke/ckpts/114688/runner_config.json @@ -0,0 +1,59 @@ +{ + "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1", + "sae": { + "d_in": 768, + "d_sae": 16384, + "dtype": "float32", + "device": "cuda", + "apply_b_dec_to_input": true, + "normalize_activations": "expected_average_only_in", + "reshape_activations": "none", + "metadata": { + "sae_lens_version": "6.43.0", + "sae_lens_training_version": "6.43.0" + }, + "decoder_init_norm": 0.1, + "k": 48, + "use_sparse_activations": false, + "aux_loss_coefficient": 1.0, + "rescale_acts_by_decoder_norm": true, + "architecture": "topk" + }, + "training_samples": 1000000, + "batch_size": 4096, + "lr": 0.0003, + "lr_warm_up_steps": 0, + "lr_decay_steps": 0, + "lr_scheduler_name": "constant", + "lr_end": 2.9999999999999997e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "n_restart_cycles": 1, + "device": "cuda", + "autocast_sae": false, + "autocast_data": false, + "n_checkpoints": 8, + "checkpoint_path": "runs/oracle_init_smoke/ckpts", + "save_final_checkpoint": true, + "output_path": "runs/oracle_init_smoke", + "save_synthetic_model": false, + "eval_frequency": 200000, + "eval_samples": 200000, + "run_final_eval": true, + "dead_feature_window": 1000, + "feature_sampling_window": 2000, + "n_batches_for_norm_estimate": 1000, + "sae_lens_version": "6.43.0", + "logger": { + "log_to_wandb": true, + "log_activations_store_to_wandb": false, + "log_optimizer_state_to_wandb": false, + "log_weights_to_wandb": true, + "wandb_project": "sae_lens_training", + "wandb_id": null, + "run_name": "synthetic-topk-16384-LR-0.0003", + "wandb_entity": null, + "wandb_log_frequency": 10, + "eval_every_n_wandb_logs": 100 + } +} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/225280/activation_scaler.json b/oracle_init_smoke/ckpts/225280/activation_scaler.json new file mode 100644 index 0000000000000000000000000000000000000000..71489f84b6af6cbeb92d8c3485dcc2dd40e4c375 --- /dev/null +++ b/oracle_init_smoke/ckpts/225280/activation_scaler.json @@ -0,0 +1 @@ +{"scaling_factor": 0.9888814189820523} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/225280/cfg.json b/oracle_init_smoke/ckpts/225280/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..2ae82a94121a05017b7424f4caca7141c9a59c56 --- /dev/null +++ b/oracle_init_smoke/ckpts/225280/cfg.json @@ -0,0 +1 @@ +{"d_in": 768, "d_sae": 16384, "dtype": "float32", "device": "cuda", "apply_b_dec_to_input": true, "normalize_activations": "expected_average_only_in", "reshape_activations": "none", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "decoder_init_norm": 0.1, "k": 48, "use_sparse_activations": false, "aux_loss_coefficient": 1.0, "rescale_acts_by_decoder_norm": true, "architecture": "topk"} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/225280/runner_config.json b/oracle_init_smoke/ckpts/225280/runner_config.json new file mode 100644 index 0000000000000000000000000000000000000000..36b25139fd4b0bc732412b5c1df03af29e4fa5f9 --- /dev/null +++ b/oracle_init_smoke/ckpts/225280/runner_config.json @@ -0,0 +1,59 @@ +{ + "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1", + "sae": { + "d_in": 768, + "d_sae": 16384, + "dtype": "float32", + "device": "cuda", + "apply_b_dec_to_input": true, + "normalize_activations": "expected_average_only_in", + "reshape_activations": "none", + "metadata": { + "sae_lens_version": "6.43.0", + "sae_lens_training_version": "6.43.0" + }, + "decoder_init_norm": 0.1, + "k": 48, + "use_sparse_activations": false, + "aux_loss_coefficient": 1.0, + "rescale_acts_by_decoder_norm": true, + "architecture": "topk" + }, + "training_samples": 1000000, + "batch_size": 4096, + "lr": 0.0003, + "lr_warm_up_steps": 0, + "lr_decay_steps": 0, + "lr_scheduler_name": "constant", + "lr_end": 2.9999999999999997e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "n_restart_cycles": 1, + "device": "cuda", + "autocast_sae": false, + "autocast_data": false, + "n_checkpoints": 8, + "checkpoint_path": "runs/oracle_init_smoke/ckpts", + "save_final_checkpoint": true, + "output_path": "runs/oracle_init_smoke", + "save_synthetic_model": false, + "eval_frequency": 200000, + "eval_samples": 200000, + "run_final_eval": true, + "dead_feature_window": 1000, + "feature_sampling_window": 2000, + "n_batches_for_norm_estimate": 1000, + "sae_lens_version": "6.43.0", + "logger": { + "log_to_wandb": true, + "log_activations_store_to_wandb": false, + "log_optimizer_state_to_wandb": false, + "log_weights_to_wandb": true, + "wandb_project": "sae_lens_training", + "wandb_id": null, + "run_name": "synthetic-topk-16384-LR-0.0003", + "wandb_entity": null, + "wandb_log_frequency": 10, + "eval_every_n_wandb_logs": 100 + } +} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/335872/activation_scaler.json b/oracle_init_smoke/ckpts/335872/activation_scaler.json new file mode 100644 index 0000000000000000000000000000000000000000..71489f84b6af6cbeb92d8c3485dcc2dd40e4c375 --- /dev/null +++ b/oracle_init_smoke/ckpts/335872/activation_scaler.json @@ -0,0 +1 @@ +{"scaling_factor": 0.9888814189820523} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/335872/cfg.json b/oracle_init_smoke/ckpts/335872/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..2ae82a94121a05017b7424f4caca7141c9a59c56 --- /dev/null +++ b/oracle_init_smoke/ckpts/335872/cfg.json @@ -0,0 +1 @@ +{"d_in": 768, "d_sae": 16384, "dtype": "float32", "device": "cuda", "apply_b_dec_to_input": true, "normalize_activations": "expected_average_only_in", "reshape_activations": "none", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "decoder_init_norm": 0.1, "k": 48, "use_sparse_activations": false, "aux_loss_coefficient": 1.0, "rescale_acts_by_decoder_norm": true, "architecture": "topk"} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/335872/runner_config.json b/oracle_init_smoke/ckpts/335872/runner_config.json new file mode 100644 index 0000000000000000000000000000000000000000..36b25139fd4b0bc732412b5c1df03af29e4fa5f9 --- /dev/null +++ b/oracle_init_smoke/ckpts/335872/runner_config.json @@ -0,0 +1,59 @@ +{ + "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1", + "sae": { + "d_in": 768, + "d_sae": 16384, + "dtype": "float32", + "device": "cuda", + "apply_b_dec_to_input": true, + "normalize_activations": "expected_average_only_in", + "reshape_activations": "none", + "metadata": { + "sae_lens_version": "6.43.0", + "sae_lens_training_version": "6.43.0" + }, + "decoder_init_norm": 0.1, + "k": 48, + "use_sparse_activations": false, + "aux_loss_coefficient": 1.0, + "rescale_acts_by_decoder_norm": true, + "architecture": "topk" + }, + "training_samples": 1000000, + "batch_size": 4096, + "lr": 0.0003, + "lr_warm_up_steps": 0, + "lr_decay_steps": 0, + "lr_scheduler_name": "constant", + "lr_end": 2.9999999999999997e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "n_restart_cycles": 1, + "device": "cuda", + "autocast_sae": false, + "autocast_data": false, + "n_checkpoints": 8, + "checkpoint_path": "runs/oracle_init_smoke/ckpts", + "save_final_checkpoint": true, + "output_path": "runs/oracle_init_smoke", + "save_synthetic_model": false, + "eval_frequency": 200000, + "eval_samples": 200000, + "run_final_eval": true, + "dead_feature_window": 1000, + "feature_sampling_window": 2000, + "n_batches_for_norm_estimate": 1000, + "sae_lens_version": "6.43.0", + "logger": { + "log_to_wandb": true, + "log_activations_store_to_wandb": false, + "log_optimizer_state_to_wandb": false, + "log_weights_to_wandb": true, + "wandb_project": "sae_lens_training", + "wandb_id": null, + "run_name": "synthetic-topk-16384-LR-0.0003", + "wandb_entity": null, + "wandb_log_frequency": 10, + "eval_every_n_wandb_logs": 100 + } +} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/446464/activation_scaler.json b/oracle_init_smoke/ckpts/446464/activation_scaler.json new file mode 100644 index 0000000000000000000000000000000000000000..71489f84b6af6cbeb92d8c3485dcc2dd40e4c375 --- /dev/null +++ b/oracle_init_smoke/ckpts/446464/activation_scaler.json @@ -0,0 +1 @@ +{"scaling_factor": 0.9888814189820523} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/446464/cfg.json b/oracle_init_smoke/ckpts/446464/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..2ae82a94121a05017b7424f4caca7141c9a59c56 --- /dev/null +++ b/oracle_init_smoke/ckpts/446464/cfg.json @@ -0,0 +1 @@ +{"d_in": 768, "d_sae": 16384, "dtype": "float32", "device": "cuda", "apply_b_dec_to_input": true, "normalize_activations": "expected_average_only_in", "reshape_activations": "none", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "decoder_init_norm": 0.1, "k": 48, "use_sparse_activations": false, "aux_loss_coefficient": 1.0, "rescale_acts_by_decoder_norm": true, "architecture": "topk"} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/446464/runner_config.json b/oracle_init_smoke/ckpts/446464/runner_config.json new file mode 100644 index 0000000000000000000000000000000000000000..36b25139fd4b0bc732412b5c1df03af29e4fa5f9 --- /dev/null +++ b/oracle_init_smoke/ckpts/446464/runner_config.json @@ -0,0 +1,59 @@ +{ + "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1", + "sae": { + "d_in": 768, + "d_sae": 16384, + "dtype": "float32", + "device": "cuda", + "apply_b_dec_to_input": true, + "normalize_activations": "expected_average_only_in", + "reshape_activations": "none", + "metadata": { + "sae_lens_version": "6.43.0", + "sae_lens_training_version": "6.43.0" + }, + "decoder_init_norm": 0.1, + "k": 48, + "use_sparse_activations": false, + "aux_loss_coefficient": 1.0, + "rescale_acts_by_decoder_norm": true, + "architecture": "topk" + }, + "training_samples": 1000000, + "batch_size": 4096, + "lr": 0.0003, + "lr_warm_up_steps": 0, + "lr_decay_steps": 0, + "lr_scheduler_name": "constant", + "lr_end": 2.9999999999999997e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "n_restart_cycles": 1, + "device": "cuda", + "autocast_sae": false, + "autocast_data": false, + "n_checkpoints": 8, + "checkpoint_path": "runs/oracle_init_smoke/ckpts", + "save_final_checkpoint": true, + "output_path": "runs/oracle_init_smoke", + "save_synthetic_model": false, + "eval_frequency": 200000, + "eval_samples": 200000, + "run_final_eval": true, + "dead_feature_window": 1000, + "feature_sampling_window": 2000, + "n_batches_for_norm_estimate": 1000, + "sae_lens_version": "6.43.0", + "logger": { + "log_to_wandb": true, + "log_activations_store_to_wandb": false, + "log_optimizer_state_to_wandb": false, + "log_weights_to_wandb": true, + "wandb_project": "sae_lens_training", + "wandb_id": null, + "run_name": "synthetic-topk-16384-LR-0.0003", + "wandb_entity": null, + "wandb_log_frequency": 10, + "eval_every_n_wandb_logs": 100 + } +} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/557056/activation_scaler.json b/oracle_init_smoke/ckpts/557056/activation_scaler.json new file mode 100644 index 0000000000000000000000000000000000000000..71489f84b6af6cbeb92d8c3485dcc2dd40e4c375 --- /dev/null +++ b/oracle_init_smoke/ckpts/557056/activation_scaler.json @@ -0,0 +1 @@ +{"scaling_factor": 0.9888814189820523} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/557056/cfg.json b/oracle_init_smoke/ckpts/557056/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..2ae82a94121a05017b7424f4caca7141c9a59c56 --- /dev/null +++ b/oracle_init_smoke/ckpts/557056/cfg.json @@ -0,0 +1 @@ +{"d_in": 768, "d_sae": 16384, "dtype": "float32", "device": "cuda", "apply_b_dec_to_input": true, "normalize_activations": "expected_average_only_in", "reshape_activations": "none", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "decoder_init_norm": 0.1, "k": 48, "use_sparse_activations": false, "aux_loss_coefficient": 1.0, "rescale_acts_by_decoder_norm": true, "architecture": "topk"} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/557056/runner_config.json b/oracle_init_smoke/ckpts/557056/runner_config.json new file mode 100644 index 0000000000000000000000000000000000000000..36b25139fd4b0bc732412b5c1df03af29e4fa5f9 --- /dev/null +++ b/oracle_init_smoke/ckpts/557056/runner_config.json @@ -0,0 +1,59 @@ +{ + "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1", + "sae": { + "d_in": 768, + "d_sae": 16384, + "dtype": "float32", + "device": "cuda", + "apply_b_dec_to_input": true, + "normalize_activations": "expected_average_only_in", + "reshape_activations": "none", + "metadata": { + "sae_lens_version": "6.43.0", + "sae_lens_training_version": "6.43.0" + }, + "decoder_init_norm": 0.1, + "k": 48, + "use_sparse_activations": false, + "aux_loss_coefficient": 1.0, + "rescale_acts_by_decoder_norm": true, + "architecture": "topk" + }, + "training_samples": 1000000, + "batch_size": 4096, + "lr": 0.0003, + "lr_warm_up_steps": 0, + "lr_decay_steps": 0, + "lr_scheduler_name": "constant", + "lr_end": 2.9999999999999997e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "n_restart_cycles": 1, + "device": "cuda", + "autocast_sae": false, + "autocast_data": false, + "n_checkpoints": 8, + "checkpoint_path": "runs/oracle_init_smoke/ckpts", + "save_final_checkpoint": true, + "output_path": "runs/oracle_init_smoke", + "save_synthetic_model": false, + "eval_frequency": 200000, + "eval_samples": 200000, + "run_final_eval": true, + "dead_feature_window": 1000, + "feature_sampling_window": 2000, + "n_batches_for_norm_estimate": 1000, + "sae_lens_version": "6.43.0", + "logger": { + "log_to_wandb": true, + "log_activations_store_to_wandb": false, + "log_optimizer_state_to_wandb": false, + "log_weights_to_wandb": true, + "wandb_project": "sae_lens_training", + "wandb_id": null, + "run_name": "synthetic-topk-16384-LR-0.0003", + "wandb_entity": null, + "wandb_log_frequency": 10, + "eval_every_n_wandb_logs": 100 + } +} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/667648/activation_scaler.json b/oracle_init_smoke/ckpts/667648/activation_scaler.json new file mode 100644 index 0000000000000000000000000000000000000000..71489f84b6af6cbeb92d8c3485dcc2dd40e4c375 --- /dev/null +++ b/oracle_init_smoke/ckpts/667648/activation_scaler.json @@ -0,0 +1 @@ +{"scaling_factor": 0.9888814189820523} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/667648/cfg.json b/oracle_init_smoke/ckpts/667648/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..2ae82a94121a05017b7424f4caca7141c9a59c56 --- /dev/null +++ b/oracle_init_smoke/ckpts/667648/cfg.json @@ -0,0 +1 @@ +{"d_in": 768, "d_sae": 16384, "dtype": "float32", "device": "cuda", "apply_b_dec_to_input": true, "normalize_activations": "expected_average_only_in", "reshape_activations": "none", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "decoder_init_norm": 0.1, "k": 48, "use_sparse_activations": false, "aux_loss_coefficient": 1.0, "rescale_acts_by_decoder_norm": true, "architecture": "topk"} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/667648/runner_config.json b/oracle_init_smoke/ckpts/667648/runner_config.json new file mode 100644 index 0000000000000000000000000000000000000000..36b25139fd4b0bc732412b5c1df03af29e4fa5f9 --- /dev/null +++ b/oracle_init_smoke/ckpts/667648/runner_config.json @@ -0,0 +1,59 @@ +{ + "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1", + "sae": { + "d_in": 768, + "d_sae": 16384, + "dtype": "float32", + "device": "cuda", + "apply_b_dec_to_input": true, + "normalize_activations": "expected_average_only_in", + "reshape_activations": "none", + "metadata": { + "sae_lens_version": "6.43.0", + "sae_lens_training_version": "6.43.0" + }, + "decoder_init_norm": 0.1, + "k": 48, + "use_sparse_activations": false, + "aux_loss_coefficient": 1.0, + "rescale_acts_by_decoder_norm": true, + "architecture": "topk" + }, + "training_samples": 1000000, + "batch_size": 4096, + "lr": 0.0003, + "lr_warm_up_steps": 0, + "lr_decay_steps": 0, + "lr_scheduler_name": "constant", + "lr_end": 2.9999999999999997e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "n_restart_cycles": 1, + "device": "cuda", + "autocast_sae": false, + "autocast_data": false, + "n_checkpoints": 8, + "checkpoint_path": "runs/oracle_init_smoke/ckpts", + "save_final_checkpoint": true, + "output_path": "runs/oracle_init_smoke", + "save_synthetic_model": false, + "eval_frequency": 200000, + "eval_samples": 200000, + "run_final_eval": true, + "dead_feature_window": 1000, + "feature_sampling_window": 2000, + "n_batches_for_norm_estimate": 1000, + "sae_lens_version": "6.43.0", + "logger": { + "log_to_wandb": true, + "log_activations_store_to_wandb": false, + "log_optimizer_state_to_wandb": false, + "log_weights_to_wandb": true, + "wandb_project": "sae_lens_training", + "wandb_id": null, + "run_name": "synthetic-topk-16384-LR-0.0003", + "wandb_entity": null, + "wandb_log_frequency": 10, + "eval_every_n_wandb_logs": 100 + } +} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/778240/activation_scaler.json b/oracle_init_smoke/ckpts/778240/activation_scaler.json new file mode 100644 index 0000000000000000000000000000000000000000..71489f84b6af6cbeb92d8c3485dcc2dd40e4c375 --- /dev/null +++ b/oracle_init_smoke/ckpts/778240/activation_scaler.json @@ -0,0 +1 @@ +{"scaling_factor": 0.9888814189820523} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/778240/cfg.json b/oracle_init_smoke/ckpts/778240/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..2ae82a94121a05017b7424f4caca7141c9a59c56 --- /dev/null +++ b/oracle_init_smoke/ckpts/778240/cfg.json @@ -0,0 +1 @@ +{"d_in": 768, "d_sae": 16384, "dtype": "float32", "device": "cuda", "apply_b_dec_to_input": true, "normalize_activations": "expected_average_only_in", "reshape_activations": "none", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "decoder_init_norm": 0.1, "k": 48, "use_sparse_activations": false, "aux_loss_coefficient": 1.0, "rescale_acts_by_decoder_norm": true, "architecture": "topk"} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/778240/runner_config.json b/oracle_init_smoke/ckpts/778240/runner_config.json new file mode 100644 index 0000000000000000000000000000000000000000..36b25139fd4b0bc732412b5c1df03af29e4fa5f9 --- /dev/null +++ b/oracle_init_smoke/ckpts/778240/runner_config.json @@ -0,0 +1,59 @@ +{ + "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1", + "sae": { + "d_in": 768, + "d_sae": 16384, + "dtype": "float32", + "device": "cuda", + "apply_b_dec_to_input": true, + "normalize_activations": "expected_average_only_in", + "reshape_activations": "none", + "metadata": { + "sae_lens_version": "6.43.0", + "sae_lens_training_version": "6.43.0" + }, + "decoder_init_norm": 0.1, + "k": 48, + "use_sparse_activations": false, + "aux_loss_coefficient": 1.0, + "rescale_acts_by_decoder_norm": true, + "architecture": "topk" + }, + "training_samples": 1000000, + "batch_size": 4096, + "lr": 0.0003, + "lr_warm_up_steps": 0, + "lr_decay_steps": 0, + "lr_scheduler_name": "constant", + "lr_end": 2.9999999999999997e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "n_restart_cycles": 1, + "device": "cuda", + "autocast_sae": false, + "autocast_data": false, + "n_checkpoints": 8, + "checkpoint_path": "runs/oracle_init_smoke/ckpts", + "save_final_checkpoint": true, + "output_path": "runs/oracle_init_smoke", + "save_synthetic_model": false, + "eval_frequency": 200000, + "eval_samples": 200000, + "run_final_eval": true, + "dead_feature_window": 1000, + "feature_sampling_window": 2000, + "n_batches_for_norm_estimate": 1000, + "sae_lens_version": "6.43.0", + "logger": { + "log_to_wandb": true, + "log_activations_store_to_wandb": false, + "log_optimizer_state_to_wandb": false, + "log_weights_to_wandb": true, + "wandb_project": "sae_lens_training", + "wandb_id": null, + "run_name": "synthetic-topk-16384-LR-0.0003", + "wandb_entity": null, + "wandb_log_frequency": 10, + "eval_every_n_wandb_logs": 100 + } +} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/892928/activation_scaler.json b/oracle_init_smoke/ckpts/892928/activation_scaler.json new file mode 100644 index 0000000000000000000000000000000000000000..71489f84b6af6cbeb92d8c3485dcc2dd40e4c375 --- /dev/null +++ b/oracle_init_smoke/ckpts/892928/activation_scaler.json @@ -0,0 +1 @@ +{"scaling_factor": 0.9888814189820523} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/892928/cfg.json b/oracle_init_smoke/ckpts/892928/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..2ae82a94121a05017b7424f4caca7141c9a59c56 --- /dev/null +++ b/oracle_init_smoke/ckpts/892928/cfg.json @@ -0,0 +1 @@ +{"d_in": 768, "d_sae": 16384, "dtype": "float32", "device": "cuda", "apply_b_dec_to_input": true, "normalize_activations": "expected_average_only_in", "reshape_activations": "none", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "decoder_init_norm": 0.1, "k": 48, "use_sparse_activations": false, "aux_loss_coefficient": 1.0, "rescale_acts_by_decoder_norm": true, "architecture": "topk"} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/892928/runner_config.json b/oracle_init_smoke/ckpts/892928/runner_config.json new file mode 100644 index 0000000000000000000000000000000000000000..36b25139fd4b0bc732412b5c1df03af29e4fa5f9 --- /dev/null +++ b/oracle_init_smoke/ckpts/892928/runner_config.json @@ -0,0 +1,59 @@ +{ + "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1", + "sae": { + "d_in": 768, + "d_sae": 16384, + "dtype": "float32", + "device": "cuda", + "apply_b_dec_to_input": true, + "normalize_activations": "expected_average_only_in", + "reshape_activations": "none", + "metadata": { + "sae_lens_version": "6.43.0", + "sae_lens_training_version": "6.43.0" + }, + "decoder_init_norm": 0.1, + "k": 48, + "use_sparse_activations": false, + "aux_loss_coefficient": 1.0, + "rescale_acts_by_decoder_norm": true, + "architecture": "topk" + }, + "training_samples": 1000000, + "batch_size": 4096, + "lr": 0.0003, + "lr_warm_up_steps": 0, + "lr_decay_steps": 0, + "lr_scheduler_name": "constant", + "lr_end": 2.9999999999999997e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "n_restart_cycles": 1, + "device": "cuda", + "autocast_sae": false, + "autocast_data": false, + "n_checkpoints": 8, + "checkpoint_path": "runs/oracle_init_smoke/ckpts", + "save_final_checkpoint": true, + "output_path": "runs/oracle_init_smoke", + "save_synthetic_model": false, + "eval_frequency": 200000, + "eval_samples": 200000, + "run_final_eval": true, + "dead_feature_window": 1000, + "feature_sampling_window": 2000, + "n_batches_for_norm_estimate": 1000, + "sae_lens_version": "6.43.0", + "logger": { + "log_to_wandb": true, + "log_activations_store_to_wandb": false, + "log_optimizer_state_to_wandb": false, + "log_weights_to_wandb": true, + "wandb_project": "sae_lens_training", + "wandb_id": null, + "run_name": "synthetic-topk-16384-LR-0.0003", + "wandb_entity": null, + "wandb_log_frequency": 10, + "eval_every_n_wandb_logs": 100 + } +} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/final_1003520/activation_scaler.json b/oracle_init_smoke/ckpts/final_1003520/activation_scaler.json new file mode 100644 index 0000000000000000000000000000000000000000..5df4fdd0c0c44542391c33143ec07c0b30ec164d --- /dev/null +++ b/oracle_init_smoke/ckpts/final_1003520/activation_scaler.json @@ -0,0 +1 @@ +{"scaling_factor": null} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/final_1003520/cfg.json b/oracle_init_smoke/ckpts/final_1003520/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..5aa7c408c3f7669833f5664a7cc637af992b7c3b --- /dev/null +++ b/oracle_init_smoke/ckpts/final_1003520/cfg.json @@ -0,0 +1 @@ +{"d_in": 768, "d_sae": 16384, "dtype": "float32", "device": "cuda", "apply_b_dec_to_input": true, "normalize_activations": "none", "reshape_activations": "none", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "decoder_init_norm": 0.1, "k": 48, "use_sparse_activations": false, "aux_loss_coefficient": 1.0, "rescale_acts_by_decoder_norm": true, "architecture": "topk"} \ No newline at end of file diff --git a/oracle_init_smoke/ckpts/final_1003520/runner_config.json b/oracle_init_smoke/ckpts/final_1003520/runner_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6a9d349a695a47644853f429ce893721a20d89dc --- /dev/null +++ b/oracle_init_smoke/ckpts/final_1003520/runner_config.json @@ -0,0 +1,59 @@ +{ + "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1", + "sae": { + "d_in": 768, + "d_sae": 16384, + "dtype": "float32", + "device": "cuda", + "apply_b_dec_to_input": true, + "normalize_activations": "none", + "reshape_activations": "none", + "metadata": { + "sae_lens_version": "6.43.0", + "sae_lens_training_version": "6.43.0" + }, + "decoder_init_norm": 0.1, + "k": 48, + "use_sparse_activations": false, + "aux_loss_coefficient": 1.0, + "rescale_acts_by_decoder_norm": true, + "architecture": "topk" + }, + "training_samples": 1000000, + "batch_size": 4096, + "lr": 0.0003, + "lr_warm_up_steps": 0, + "lr_decay_steps": 0, + "lr_scheduler_name": "constant", + "lr_end": 2.9999999999999997e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "n_restart_cycles": 1, + "device": "cuda", + "autocast_sae": false, + "autocast_data": false, + "n_checkpoints": 8, + "checkpoint_path": "runs/oracle_init_smoke/ckpts", + "save_final_checkpoint": true, + "output_path": "runs/oracle_init_smoke", + "save_synthetic_model": false, + "eval_frequency": 200000, + "eval_samples": 200000, + "run_final_eval": true, + "dead_feature_window": 1000, + "feature_sampling_window": 2000, + "n_batches_for_norm_estimate": 1000, + "sae_lens_version": "6.43.0", + "logger": { + "log_to_wandb": true, + "log_activations_store_to_wandb": false, + "log_optimizer_state_to_wandb": false, + "log_weights_to_wandb": true, + "wandb_project": "sae_lens_training", + "wandb_id": null, + "run_name": "synthetic-topk-16384-LR-0.0003", + "wandb_entity": null, + "wandb_log_frequency": 10, + "eval_every_n_wandb_logs": 100 + } +} \ No newline at end of file diff --git a/oracle_init_smoke/eval_stats.json b/oracle_init_smoke/eval_stats.json new file mode 100644 index 0000000000000000000000000000000000000000..604bcd74fb841a63a4c25be0701ce317dfb2cf36 --- /dev/null +++ b/oracle_init_smoke/eval_stats.json @@ -0,0 +1,15 @@ +{ + "true_l0": 34.46811, + "sae_l0": 47.999505, + "dead_latents": 0, + "shrinkage": 0.9734420861816406, + "explained_variance": 0.9196613902159156, + "mcc": 0.990073561668396, + "uniqueness": 1.0, + "classification": { + "precision": 0.5700647234916687, + "recall": 0.91277015209198, + "f1_score": 0.6972928047180176, + "accuracy": 0.998992383480072 + } +} \ No newline at end of file diff --git a/oracle_init_smoke/meta.json b/oracle_init_smoke/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..6974304c7a0083e9c128d6c861faef3531dce10c --- /dev/null +++ b/oracle_init_smoke/meta.json @@ -0,0 +1,8 @@ +{ + "k": 48, + "seed": 1, + "n_samples": 1000000, + "cos_init": 1.0, + "cos_final": 0.990073561668396, + "seconds": 1409.8078198432922 +} \ No newline at end of file diff --git a/oracle_init_smoke/runner_config.json b/oracle_init_smoke/runner_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6a9d349a695a47644853f429ce893721a20d89dc --- /dev/null +++ b/oracle_init_smoke/runner_config.json @@ -0,0 +1,59 @@ +{ + "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1", + "sae": { + "d_in": 768, + "d_sae": 16384, + "dtype": "float32", + "device": "cuda", + "apply_b_dec_to_input": true, + "normalize_activations": "none", + "reshape_activations": "none", + "metadata": { + "sae_lens_version": "6.43.0", + "sae_lens_training_version": "6.43.0" + }, + "decoder_init_norm": 0.1, + "k": 48, + "use_sparse_activations": false, + "aux_loss_coefficient": 1.0, + "rescale_acts_by_decoder_norm": true, + "architecture": "topk" + }, + "training_samples": 1000000, + "batch_size": 4096, + "lr": 0.0003, + "lr_warm_up_steps": 0, + "lr_decay_steps": 0, + "lr_scheduler_name": "constant", + "lr_end": 2.9999999999999997e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "n_restart_cycles": 1, + "device": "cuda", + "autocast_sae": false, + "autocast_data": false, + "n_checkpoints": 8, + "checkpoint_path": "runs/oracle_init_smoke/ckpts", + "save_final_checkpoint": true, + "output_path": "runs/oracle_init_smoke", + "save_synthetic_model": false, + "eval_frequency": 200000, + "eval_samples": 200000, + "run_final_eval": true, + "dead_feature_window": 1000, + "feature_sampling_window": 2000, + "n_batches_for_norm_estimate": 1000, + "sae_lens_version": "6.43.0", + "logger": { + "log_to_wandb": true, + "log_activations_store_to_wandb": false, + "log_optimizer_state_to_wandb": false, + "log_weights_to_wandb": true, + "wandb_project": "sae_lens_training", + "wandb_id": null, + "run_name": "synthetic-topk-16384-LR-0.0003", + "wandb_entity": null, + "wandb_log_frequency": 10, + "eval_every_n_wandb_logs": 100 + } +} \ No newline at end of file diff --git a/p2_warmup_k48_s1/results.json b/p2_warmup_k48_s1/results.json new file mode 100644 index 0000000000000000000000000000000000000000..a5bfd0194b1f131baef310c24c0adfe6d7f40950 --- /dev/null +++ b/p2_warmup_k48_s1/results.json @@ -0,0 +1,19 @@ +{ + "variant": "warmup", + "k": 48, + "n_samples": 30000000, + "warmup_samples": 5000000, + "lr_enc": 0.0003, + "lr_dec": 3e-05, + "seed": 1, + "eval": { + "true_l0": 34.48436, + "sae_l0": 48.0, + "dead_latents": 3371, + "shrinkage": 0.9116807043457031, + "explained_variance": 0.8029508913346035, + "mcc": 0.3172292709350586, + "uniqueness": 0.52288818359375 + }, + "seconds": 512.8025069236755 +} \ No newline at end of file diff --git a/topk_k35_s1/barrier.json b/topk_k35_s1/barrier.json new file mode 100644 index 0000000000000000000000000000000000000000..a1018b83acb9c715ca54287a4cfee6917dfc95fc --- /dev/null +++ b/topk_k35_s1/barrier.json @@ -0,0 +1,52 @@ +{ + "run": "runs/topk_k35_s1", + "k": 35, + "n_samples": 5000, + "aligned_cos_mean": 0.6303648948669434, + "alphas": [ + 0.0, + 0.05, + 0.1, + 0.15000000000000002, + 0.2, + 0.25, + 0.30000000000000004, + 0.35000000000000003, + 0.4, + 0.45, + 0.5, + 0.55, + 0.6000000000000001, + 0.65, + 0.7000000000000001, + 0.75, + 0.8, + 0.8500000000000001, + 0.9, + 0.9500000000000001, + 1.0 + ], + "mse": [ + 0.0453842468559742, + 0.04573005065321922, + 0.04693974554538727, + 0.04897211119532585, + 0.05176183208823204, + 0.0552293062210083, + 0.05915657803416252, + 0.06315315514802933, + 0.06675059348344803, + 0.06954248994588852, + 0.0716228038072586, + 0.07325062155723572, + 0.07455261796712875, + 0.07554967701435089, + 0.07634808123111725, + 0.07705218344926834, + 0.07750643044710159, + 0.0778859406709671, + 0.07841446995735168, + 0.07943202555179596, + 0.08122687041759491 + ] +} \ No newline at end of file diff --git a/topk_k35_s1/cfg.json b/topk_k35_s1/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..46ac961471a9b8b398d8ede94ac83415eca3c572 --- /dev/null +++ b/topk_k35_s1/cfg.json @@ -0,0 +1 @@ +{"device": "cuda", "rescale_acts_by_decoder_norm": true, "dtype": "float32", "d_in": 768, "reshape_activations": "none", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "k": 35, "normalize_activations": "none", "d_sae": 16384, "apply_b_dec_to_input": true, "architecture": "topk"} \ No newline at end of file diff --git a/topk_k35_s1/ckpts/final_50003968/activation_scaler.json b/topk_k35_s1/ckpts/final_50003968/activation_scaler.json new file mode 100644 index 0000000000000000000000000000000000000000..5df4fdd0c0c44542391c33143ec07c0b30ec164d --- /dev/null +++ b/topk_k35_s1/ckpts/final_50003968/activation_scaler.json @@ -0,0 +1 @@ +{"scaling_factor": null} \ No newline at end of file diff --git a/topk_k35_s1/ckpts/final_50003968/cfg.json b/topk_k35_s1/ckpts/final_50003968/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0d385a65196a9a7a30994e6fbdb105480b0199f8 --- /dev/null +++ b/topk_k35_s1/ckpts/final_50003968/cfg.json @@ -0,0 +1 @@ +{"d_in": 768, "d_sae": 16384, "dtype": "float32", "device": "cuda", "apply_b_dec_to_input": true, "normalize_activations": "none", "reshape_activations": "none", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "decoder_init_norm": 0.1, "k": 35, "use_sparse_activations": false, "aux_loss_coefficient": 1.0, "rescale_acts_by_decoder_norm": true, "architecture": "topk"} \ No newline at end of file diff --git a/topk_k35_s1/ckpts/final_50003968/runner_config.json b/topk_k35_s1/ckpts/final_50003968/runner_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e74249d0ad4e6e884d369119d4342786b81a9ece --- /dev/null +++ b/topk_k35_s1/ckpts/final_50003968/runner_config.json @@ -0,0 +1,59 @@ +{ + "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1", + "sae": { + "d_in": 768, + "d_sae": 16384, + "dtype": "float32", + "device": "cuda", + "apply_b_dec_to_input": true, + "normalize_activations": "none", + "reshape_activations": "none", + "metadata": { + "sae_lens_version": "6.43.0", + "sae_lens_training_version": "6.43.0" + }, + "decoder_init_norm": 0.1, + "k": 35, + "use_sparse_activations": false, + "aux_loss_coefficient": 1.0, + "rescale_acts_by_decoder_norm": true, + "architecture": "topk" + }, + "training_samples": 50000000, + "batch_size": 4096, + "lr": 0.0003, + "lr_warm_up_steps": 1000, + "lr_decay_steps": 0, + "lr_scheduler_name": "constant", + "lr_end": 2.9999999999999997e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "n_restart_cycles": 1, + "device": "cuda", + "autocast_sae": false, + "autocast_data": false, + "n_checkpoints": 0, + "checkpoint_path": "runs/topk_k35_s1/ckpts", + "save_final_checkpoint": true, + "output_path": "runs/topk_k35_s1", + "save_synthetic_model": false, + "eval_frequency": 200000, + "eval_samples": 200000, + "run_final_eval": true, + "dead_feature_window": 1000, + "feature_sampling_window": 2000, + "n_batches_for_norm_estimate": 1000, + "sae_lens_version": "6.43.0", + "logger": { + "log_to_wandb": true, + "log_activations_store_to_wandb": false, + "log_optimizer_state_to_wandb": false, + "log_weights_to_wandb": true, + "wandb_project": "sae_lens_training", + "wandb_id": null, + "run_name": "synthetic-topk-16384-LR-0.0003", + "wandb_entity": null, + "wandb_log_frequency": 10, + "eval_every_n_wandb_logs": 100 + } +} \ No newline at end of file diff --git a/topk_k35_s1/eval_stats.json b/topk_k35_s1/eval_stats.json new file mode 100644 index 0000000000000000000000000000000000000000..fa6835f11a9ddf380f9e7be6a089a696848294ce --- /dev/null +++ b/topk_k35_s1/eval_stats.json @@ -0,0 +1,15 @@ +{ + "true_l0": 34.512515, + "sae_l0": 35.0, + "dead_latents": 2750, + "shrinkage": 0.9522375036621094, + "explained_variance": 0.9103770920925818, + "mcc": 0.6321330666542053, + "uniqueness": 0.7606201171875, + "classification": { + "precision": 0.4823722243309021, + "recall": 0.6607217788696289, + "f1_score": 0.5374996066093445, + "accuracy": 0.9961609840393066 + } +} \ No newline at end of file diff --git a/topk_k35_s1/meta.json b/topk_k35_s1/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c6b1a987fdecf353ffc5ebc450e179b902edd1a0 --- /dev/null +++ b/topk_k35_s1/meta.json @@ -0,0 +1,9 @@ +{ + "variant": "topk", + "k": 35, + "l1": 5.0, + "seed": 1, + "n_samples": 50000000, + "seconds": 1490.9028396606445, + "out": "runs/topk_k35_s1" +} \ No newline at end of file diff --git a/topk_k35_s1/metrics.json b/topk_k35_s1/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..add4fe8bddb1bdc2570a30e756d599b340d6f4cc --- /dev/null +++ b/topk_k35_s1/metrics.json @@ -0,0 +1,20 @@ +{ + "geometric": { + "max_cos_mean": 0.6783078908920288, + "max_cos_median": 0.7684886455535889, + "frac_recovered_0.9": 0.22515869140625, + "frac_recovered_0.95": 0.18798828125, + "hungarian_mean": 0.6303648948669434, + "hungarian_frac_0.9": 0.22515869140625 + }, + "sae_lens_eval": { + "true_l0": 34.48769, + "sae_l0": 35.0, + "dead_latents": 2111, + "shrinkage": 0.9522427712402344, + "explained_variance": 0.9104231035541497, + "mcc": 0.6321330666542053, + "uniqueness": 0.7606201171875, + "classification": "" + } +} \ No newline at end of file diff --git a/topk_k35_s1/mode_connectivity.json b/topk_k35_s1/mode_connectivity.json new file mode 100644 index 0000000000000000000000000000000000000000..000fe8bfc82e2a47667b756f47b9108ee6a08278 --- /dev/null +++ b/topk_k35_s1/mode_connectivity.json @@ -0,0 +1,57 @@ +{ + "run": "runs/topk_k35_s1", + "k": 35, + "train_alphas": [ + 0.2, + 0.4, + 0.5, + 0.6, + 0.8 + ], + "alphas": [ + 0.0, + 0.05, + 0.1, + 0.15000000000000002, + 0.2, + 0.25, + 0.30000000000000004, + 0.35000000000000003, + 0.4, + 0.45, + 0.5, + 0.55, + 0.6000000000000001, + 0.65, + 0.7000000000000001, + 0.75, + 0.8, + 0.8500000000000001, + 0.9, + 0.9500000000000001, + 1.0 + ], + "optimised_mse": [ + 0.04496241733431816, + 0.04562588036060333, + 0.04834703728556633, + 0.05097825825214386, + 0.05279238894581795, + 0.054847173392772675, + 0.056604404002428055, + 0.05785268172621727, + 0.05841264873743057, + 0.05831972882151604, + 0.05767783895134926, + 0.05667198821902275, + 0.05547071993350983, + 0.05442268028855324, + 0.0540359802544117, + 0.054819148033857346, + 0.05726069584488869, + 0.061471112072467804, + 0.06672156602144241, + 0.0717395693063736, + 0.07593071460723877 + ] +} \ No newline at end of file diff --git a/topk_k35_s1/runner_config.json b/topk_k35_s1/runner_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e74249d0ad4e6e884d369119d4342786b81a9ece --- /dev/null +++ b/topk_k35_s1/runner_config.json @@ -0,0 +1,59 @@ +{ + "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1", + "sae": { + "d_in": 768, + "d_sae": 16384, + "dtype": "float32", + "device": "cuda", + "apply_b_dec_to_input": true, + "normalize_activations": "none", + "reshape_activations": "none", + "metadata": { + "sae_lens_version": "6.43.0", + "sae_lens_training_version": "6.43.0" + }, + "decoder_init_norm": 0.1, + "k": 35, + "use_sparse_activations": false, + "aux_loss_coefficient": 1.0, + "rescale_acts_by_decoder_norm": true, + "architecture": "topk" + }, + "training_samples": 50000000, + "batch_size": 4096, + "lr": 0.0003, + "lr_warm_up_steps": 1000, + "lr_decay_steps": 0, + "lr_scheduler_name": "constant", + "lr_end": 2.9999999999999997e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "n_restart_cycles": 1, + "device": "cuda", + "autocast_sae": false, + "autocast_data": false, + "n_checkpoints": 0, + "checkpoint_path": "runs/topk_k35_s1/ckpts", + "save_final_checkpoint": true, + "output_path": "runs/topk_k35_s1", + "save_synthetic_model": false, + "eval_frequency": 200000, + "eval_samples": 200000, + "run_final_eval": true, + "dead_feature_window": 1000, + "feature_sampling_window": 2000, + "n_batches_for_norm_estimate": 1000, + "sae_lens_version": "6.43.0", + "logger": { + "log_to_wandb": true, + "log_activations_store_to_wandb": false, + "log_optimizer_state_to_wandb": false, + "log_weights_to_wandb": true, + "wandb_project": "sae_lens_training", + "wandb_id": null, + "run_name": "synthetic-topk-16384-LR-0.0003", + "wandb_entity": null, + "wandb_log_frequency": 10, + "eval_every_n_wandb_logs": 100 + } +} \ No newline at end of file diff --git a/topk_k35_s2/cfg.json b/topk_k35_s2/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..f5c407d6b0f9e79f26005053695604c7e823461e --- /dev/null +++ b/topk_k35_s2/cfg.json @@ -0,0 +1 @@ +{"d_sae": 16384, "d_in": 768, "reshape_activations": "none", "normalize_activations": "none", "rescale_acts_by_decoder_norm": true, "dtype": "float32", "k": 35, "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "apply_b_dec_to_input": true, "device": "cuda", "architecture": "topk"} \ No newline at end of file diff --git a/topk_k35_s2/meta.json b/topk_k35_s2/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..2bf3c91a2a9f860bd384329e44cf918d1d4e8012 --- /dev/null +++ b/topk_k35_s2/meta.json @@ -0,0 +1,9 @@ +{ + "variant": "topk", + "k": 35, + "l1": 5.0, + "seed": 2, + "n_samples": 50000000, + "seconds": 1482.5321309566498, + "out": "runs/topk_k35_s2" +} \ No newline at end of file diff --git a/topk_k35_s2/mode_connectivity.json b/topk_k35_s2/mode_connectivity.json new file mode 100644 index 0000000000000000000000000000000000000000..ca19e586757104a2e249f0aa628772b98cf47ffc --- /dev/null +++ b/topk_k35_s2/mode_connectivity.json @@ -0,0 +1,57 @@ +{ + "run": "runs/topk_k35_s2", + "k": 35, + "train_alphas": [ + 0.2, + 0.4, + 0.5, + 0.6, + 0.8 + ], + "alphas": [ + 0.0, + 0.05, + 0.1, + 0.15000000000000002, + 0.2, + 0.25, + 0.30000000000000004, + 0.35000000000000003, + 0.4, + 0.45, + 0.5, + 0.55, + 0.6000000000000001, + 0.65, + 0.7000000000000001, + 0.75, + 0.8, + 0.8500000000000001, + 0.9, + 0.9500000000000001, + 1.0 + ], + "optimised_mse": [ + 0.04629234969615936, + 0.04698051139712334, + 0.049797359853982925, + 0.05216974392533302, + 0.05400341376662254, + 0.0560045950114727, + 0.05774642899632454, + 0.05896490439772606, + 0.059549152851104736, + 0.05950948968529701, + 0.058909233659505844, + 0.057968396693468094, + 0.05693399906158447, + 0.056061241775751114, + 0.05586361512541771, + 0.05694751814007759, + 0.0596652515232563, + 0.06424006074666977, + 0.0700952336192131, + 0.0754917711019516, + 0.07991532981395721 + ] +} \ No newline at end of file diff --git a/topk_k40_s2/barrier.json b/topk_k40_s2/barrier.json new file mode 100644 index 0000000000000000000000000000000000000000..993c187232a5603aba36db2a78291d8092d1903e --- /dev/null +++ b/topk_k40_s2/barrier.json @@ -0,0 +1,52 @@ +{ + "run": "runs/topk_k40_s2", + "k": 40, + "n_samples": 5000, + "aligned_cos_mean": 0.6575612425804138, + "alphas": [ + 0.0, + 0.05, + 0.1, + 0.15000000000000002, + 0.2, + 0.25, + 0.30000000000000004, + 0.35000000000000003, + 0.4, + 0.45, + 0.5, + 0.55, + 0.6000000000000001, + 0.65, + 0.7000000000000001, + 0.75, + 0.8, + 0.8500000000000001, + 0.9, + 0.9500000000000001, + 1.0 + ], + "mse": [ + 0.03702162951231003, + 0.03730197250843048, + 0.0383327342569828, + 0.04001215845346451, + 0.042289502918720245, + 0.04502546414732933, + 0.04801555350422859, + 0.05084111541509628, + 0.0529969148337841, + 0.05404459685087204, + 0.053965795785188675, + 0.05313773825764656, + 0.05161897838115692, + 0.0498528890311718, + 0.047844402492046356, + 0.045882564038038254, + 0.043987590819597244, + 0.04239460453391075, + 0.04129099100828171, + 0.04094821959733963, + 0.04158172756433487 + ] +} \ No newline at end of file diff --git a/topk_k40_s2/cfg.json b/topk_k40_s2/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..ccb7be68434a8bfa42c2c1b0d9c0a0750ed034f3 --- /dev/null +++ b/topk_k40_s2/cfg.json @@ -0,0 +1 @@ +{"apply_b_dec_to_input": true, "dtype": "float32", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "rescale_acts_by_decoder_norm": true, "reshape_activations": "none", "normalize_activations": "none", "k": 40, "device": "cuda", "d_sae": 16384, "d_in": 768, "architecture": "topk"} \ No newline at end of file diff --git a/topk_k40_s2/ckpts/final_50003968/activation_scaler.json b/topk_k40_s2/ckpts/final_50003968/activation_scaler.json new file mode 100644 index 0000000000000000000000000000000000000000..5df4fdd0c0c44542391c33143ec07c0b30ec164d --- /dev/null +++ b/topk_k40_s2/ckpts/final_50003968/activation_scaler.json @@ -0,0 +1 @@ +{"scaling_factor": null} \ No newline at end of file diff --git a/topk_k40_s2/ckpts/final_50003968/cfg.json b/topk_k40_s2/ckpts/final_50003968/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..449107e65944d5751d26792a03e931db7f54bd3c --- /dev/null +++ b/topk_k40_s2/ckpts/final_50003968/cfg.json @@ -0,0 +1 @@ +{"d_in": 768, "d_sae": 16384, "dtype": "float32", "device": "cuda", "apply_b_dec_to_input": true, "normalize_activations": "none", "reshape_activations": "none", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "decoder_init_norm": 0.1, "k": 40, "use_sparse_activations": false, "aux_loss_coefficient": 1.0, "rescale_acts_by_decoder_norm": true, "architecture": "topk"} \ No newline at end of file diff --git a/topk_k40_s2/ckpts/final_50003968/runner_config.json b/topk_k40_s2/ckpts/final_50003968/runner_config.json new file mode 100644 index 0000000000000000000000000000000000000000..765451c5bf7d3c0658bb14b59f1d6da4eca63a2a --- /dev/null +++ b/topk_k40_s2/ckpts/final_50003968/runner_config.json @@ -0,0 +1,59 @@ +{ + "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1", + "sae": { + "d_in": 768, + "d_sae": 16384, + "dtype": "float32", + "device": "cuda", + "apply_b_dec_to_input": true, + "normalize_activations": "none", + "reshape_activations": "none", + "metadata": { + "sae_lens_version": "6.43.0", + "sae_lens_training_version": "6.43.0" + }, + "decoder_init_norm": 0.1, + "k": 40, + "use_sparse_activations": false, + "aux_loss_coefficient": 1.0, + "rescale_acts_by_decoder_norm": true, + "architecture": "topk" + }, + "training_samples": 50000000, + "batch_size": 4096, + "lr": 0.0003, + "lr_warm_up_steps": 1000, + "lr_decay_steps": 0, + "lr_scheduler_name": "constant", + "lr_end": 2.9999999999999997e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "n_restart_cycles": 1, + "device": "cuda", + "autocast_sae": false, + "autocast_data": false, + "n_checkpoints": 0, + "checkpoint_path": "runs/topk_k40_s2/ckpts", + "save_final_checkpoint": true, + "output_path": "runs/topk_k40_s2", + "save_synthetic_model": false, + "eval_frequency": 200000, + "eval_samples": 200000, + "run_final_eval": true, + "dead_feature_window": 1000, + "feature_sampling_window": 2000, + "n_batches_for_norm_estimate": 1000, + "sae_lens_version": "6.43.0", + "logger": { + "log_to_wandb": true, + "log_activations_store_to_wandb": false, + "log_optimizer_state_to_wandb": false, + "log_weights_to_wandb": true, + "wandb_project": "sae_lens_training", + "wandb_id": null, + "run_name": "synthetic-topk-16384-LR-0.0003", + "wandb_entity": null, + "wandb_log_frequency": 10, + "eval_every_n_wandb_logs": 100 + } +} \ No newline at end of file diff --git a/topk_k40_s2/eval_stats.json b/topk_k40_s2/eval_stats.json new file mode 100644 index 0000000000000000000000000000000000000000..2acd2b52fabc5c40ec1c57e7d6c674e669a069d6 --- /dev/null +++ b/topk_k40_s2/eval_stats.json @@ -0,0 +1,15 @@ +{ + "true_l0": 34.46119, + "sae_l0": 40.0, + "dead_latents": 2791, + "shrinkage": 0.9558141149902344, + "explained_variance": 0.9174528797345165, + "mcc": 0.6593994498252869, + "uniqueness": 0.7716064453125, + "classification": { + "precision": 0.4852391183376312, + "recall": 0.6962364315986633, + "f1_score": 0.5553758144378662, + "accuracy": 0.9972767233848572 + } +} \ No newline at end of file diff --git a/topk_k40_s2/meta.json b/topk_k40_s2/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..fca404b1ca24a4cea6d2ae3960fc6bf9f7699fe1 --- /dev/null +++ b/topk_k40_s2/meta.json @@ -0,0 +1,9 @@ +{ + "variant": "topk", + "k": 40, + "l1": 5.0, + "seed": 2, + "n_samples": 50000000, + "seconds": 1522.9690182209015, + "out": "runs/topk_k40_s2" +} \ No newline at end of file diff --git a/topk_k40_s2/mode_connectivity.json b/topk_k40_s2/mode_connectivity.json new file mode 100644 index 0000000000000000000000000000000000000000..d9adad4136f8c0bfadd36c6bd6eb6c13dd3f55b2 --- /dev/null +++ b/topk_k40_s2/mode_connectivity.json @@ -0,0 +1,57 @@ +{ + "run": "runs/topk_k40_s2", + "k": 40, + "train_alphas": [ + 0.2, + 0.4, + 0.5, + 0.6, + 0.8 + ], + "alphas": [ + 0.0, + 0.05, + 0.1, + 0.15000000000000002, + 0.2, + 0.25, + 0.30000000000000004, + 0.35000000000000003, + 0.4, + 0.45, + 0.5, + 0.55, + 0.6000000000000001, + 0.65, + 0.7000000000000001, + 0.75, + 0.8, + 0.8500000000000001, + 0.9, + 0.9500000000000001, + 1.0 + ], + "optimised_mse": [ + 0.037498991936445236, + 0.03806496039032936, + 0.04053053259849548, + 0.043738577514886856, + 0.0451018363237381, + 0.046961281448602676, + 0.04870113357901573, + 0.049923527985811234, + 0.05049281567335129, + 0.05037324130535126, + 0.049595288932323456, + 0.04825925827026367, + 0.046501725912094116, + 0.044442757964134216, + 0.04250013828277588, + 0.04095690697431564, + 0.04013577476143837, + 0.040213216096162796, + 0.04093411937355995, + 0.04172792285680771, + 0.04303701967000961 + ] +} \ No newline at end of file diff --git a/topk_k40_s2/runner_config.json b/topk_k40_s2/runner_config.json new file mode 100644 index 0000000000000000000000000000000000000000..765451c5bf7d3c0658bb14b59f1d6da4eca63a2a --- /dev/null +++ b/topk_k40_s2/runner_config.json @@ -0,0 +1,59 @@ +{ + "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1", + "sae": { + "d_in": 768, + "d_sae": 16384, + "dtype": "float32", + "device": "cuda", + "apply_b_dec_to_input": true, + "normalize_activations": "none", + "reshape_activations": "none", + "metadata": { + "sae_lens_version": "6.43.0", + "sae_lens_training_version": "6.43.0" + }, + "decoder_init_norm": 0.1, + "k": 40, + "use_sparse_activations": false, + "aux_loss_coefficient": 1.0, + "rescale_acts_by_decoder_norm": true, + "architecture": "topk" + }, + "training_samples": 50000000, + "batch_size": 4096, + "lr": 0.0003, + "lr_warm_up_steps": 1000, + "lr_decay_steps": 0, + "lr_scheduler_name": "constant", + "lr_end": 2.9999999999999997e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "n_restart_cycles": 1, + "device": "cuda", + "autocast_sae": false, + "autocast_data": false, + "n_checkpoints": 0, + "checkpoint_path": "runs/topk_k40_s2/ckpts", + "save_final_checkpoint": true, + "output_path": "runs/topk_k40_s2", + "save_synthetic_model": false, + "eval_frequency": 200000, + "eval_samples": 200000, + "run_final_eval": true, + "dead_feature_window": 1000, + "feature_sampling_window": 2000, + "n_batches_for_norm_estimate": 1000, + "sae_lens_version": "6.43.0", + "logger": { + "log_to_wandb": true, + "log_activations_store_to_wandb": false, + "log_optimizer_state_to_wandb": false, + "log_weights_to_wandb": true, + "wandb_project": "sae_lens_training", + "wandb_id": null, + "run_name": "synthetic-topk-16384-LR-0.0003", + "wandb_entity": null, + "wandb_log_frequency": 10, + "eval_every_n_wandb_logs": 100 + } +} \ No newline at end of file diff --git a/topk_k45_s1/barrier.json b/topk_k45_s1/barrier.json new file mode 100644 index 0000000000000000000000000000000000000000..545790612fed9af89d1513ee7f89ac9d1d16571f --- /dev/null +++ b/topk_k45_s1/barrier.json @@ -0,0 +1,52 @@ +{ + "run": "runs/topk_k45_s1", + "k": 45, + "n_samples": 5000, + "aligned_cos_mean": 0.6867638826370239, + "alphas": [ + 0.0, + 0.05, + 0.1, + 0.15000000000000002, + 0.2, + 0.25, + 0.30000000000000004, + 0.35000000000000003, + 0.4, + 0.45, + 0.5, + 0.55, + 0.6000000000000001, + 0.65, + 0.7000000000000001, + 0.75, + 0.8, + 0.8500000000000001, + 0.9, + 0.9500000000000001, + 1.0 + ], + "mse": [ + 0.030022021383047104, + 0.03060985542833805, + 0.03255172073841095, + 0.03565589711070061, + 0.03989682346582413, + 0.045139897614717484, + 0.05113111063838005, + 0.05728474631905556, + 0.06257577240467072, + 0.06588007509708405, + 0.06657329946756363, + 0.06489714235067368, + 0.061188843101263046, + 0.055839844048023224, + 0.04892411455512047, + 0.04163787141442299, + 0.03479789197444916, + 0.028989331796765327, + 0.024702437222003937, + 0.02213997021317482, + 0.021650614216923714 + ] +} \ No newline at end of file diff --git a/topk_k45_s1/cfg.json b/topk_k45_s1/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..88ba3c20c86477ca643ddf783e57603d564cafb8 --- /dev/null +++ b/topk_k45_s1/cfg.json @@ -0,0 +1 @@ +{"normalize_activations": "none", "rescale_acts_by_decoder_norm": true, "apply_b_dec_to_input": true, "d_in": 768, "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "d_sae": 16384, "reshape_activations": "none", "dtype": "float32", "k": 45, "device": "cuda", "architecture": "topk"} \ No newline at end of file diff --git a/topk_k45_s1/ckpts/final_50003968/activation_scaler.json b/topk_k45_s1/ckpts/final_50003968/activation_scaler.json new file mode 100644 index 0000000000000000000000000000000000000000..5df4fdd0c0c44542391c33143ec07c0b30ec164d --- /dev/null +++ b/topk_k45_s1/ckpts/final_50003968/activation_scaler.json @@ -0,0 +1 @@ +{"scaling_factor": null} \ No newline at end of file diff --git a/topk_k45_s1/ckpts/final_50003968/cfg.json b/topk_k45_s1/ckpts/final_50003968/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..2ceca08c7967ce3735763fdad20982f79e7c7193 --- /dev/null +++ b/topk_k45_s1/ckpts/final_50003968/cfg.json @@ -0,0 +1 @@ +{"d_in": 768, "d_sae": 16384, "dtype": "float32", "device": "cuda", "apply_b_dec_to_input": true, "normalize_activations": "none", "reshape_activations": "none", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "decoder_init_norm": 0.1, "k": 45, "use_sparse_activations": false, "aux_loss_coefficient": 1.0, "rescale_acts_by_decoder_norm": true, "architecture": "topk"} \ No newline at end of file diff --git a/topk_k45_s1/ckpts/final_50003968/runner_config.json b/topk_k45_s1/ckpts/final_50003968/runner_config.json new file mode 100644 index 0000000000000000000000000000000000000000..adb72d14e7822bc83104b25e7d166bd5788f445d --- /dev/null +++ b/topk_k45_s1/ckpts/final_50003968/runner_config.json @@ -0,0 +1,59 @@ +{ + "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1", + "sae": { + "d_in": 768, + "d_sae": 16384, + "dtype": "float32", + "device": "cuda", + "apply_b_dec_to_input": true, + "normalize_activations": "none", + "reshape_activations": "none", + "metadata": { + "sae_lens_version": "6.43.0", + "sae_lens_training_version": "6.43.0" + }, + "decoder_init_norm": 0.1, + "k": 45, + "use_sparse_activations": false, + "aux_loss_coefficient": 1.0, + "rescale_acts_by_decoder_norm": true, + "architecture": "topk" + }, + "training_samples": 50000000, + "batch_size": 4096, + "lr": 0.0003, + "lr_warm_up_steps": 1000, + "lr_decay_steps": 0, + "lr_scheduler_name": "constant", + "lr_end": 2.9999999999999997e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "n_restart_cycles": 1, + "device": "cuda", + "autocast_sae": false, + "autocast_data": false, + "n_checkpoints": 0, + "checkpoint_path": "runs/topk_k45_s1/ckpts", + "save_final_checkpoint": true, + "output_path": "runs/topk_k45_s1", + "save_synthetic_model": false, + "eval_frequency": 200000, + "eval_samples": 200000, + "run_final_eval": true, + "dead_feature_window": 1000, + "feature_sampling_window": 2000, + "n_batches_for_norm_estimate": 1000, + "sae_lens_version": "6.43.0", + "logger": { + "log_to_wandb": true, + "log_activations_store_to_wandb": false, + "log_optimizer_state_to_wandb": false, + "log_weights_to_wandb": true, + "wandb_project": "sae_lens_training", + "wandb_id": null, + "run_name": "synthetic-topk-16384-LR-0.0003", + "wandb_entity": null, + "wandb_log_frequency": 10, + "eval_every_n_wandb_logs": 100 + } +} \ No newline at end of file diff --git a/topk_k45_s1/eval_stats.json b/topk_k45_s1/eval_stats.json new file mode 100644 index 0000000000000000000000000000000000000000..ad8a339eb58a064b9c86106e2446a3543350881e --- /dev/null +++ b/topk_k45_s1/eval_stats.json @@ -0,0 +1,15 @@ +{ + "true_l0": 34.512515, + "sae_l0": 45.0, + "dead_latents": 2436, + "shrinkage": 0.9596410986328126, + "explained_variance": 0.9241292106221437, + "mcc": 0.6890791654586792, + "uniqueness": 0.79144287109375, + "classification": { + "precision": 0.49001747369766235, + "recall": 0.7316815257072449, + "f1_score": 0.5706678628921509, + "accuracy": 0.996722936630249 + } +} \ No newline at end of file diff --git a/topk_k45_s1/meta.json b/topk_k45_s1/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..c6f40287b356deb1550a87af23f1c6c491303cab --- /dev/null +++ b/topk_k45_s1/meta.json @@ -0,0 +1,9 @@ +{ + "variant": "topk", + "k": 45, + "l1": 5.0, + "seed": 1, + "n_samples": 50000000, + "seconds": 1574.1721713542938, + "out": "runs/topk_k45_s1" +} \ No newline at end of file diff --git a/topk_k45_s1/mode_connectivity.json b/topk_k45_s1/mode_connectivity.json new file mode 100644 index 0000000000000000000000000000000000000000..e1f7590ed22c578918513d07f804e741cb4a90ed --- /dev/null +++ b/topk_k45_s1/mode_connectivity.json @@ -0,0 +1,57 @@ +{ + "run": "runs/topk_k45_s1", + "k": 45, + "train_alphas": [ + 0.2, + 0.4, + 0.5, + 0.6, + 0.8 + ], + "alphas": [ + 0.0, + 0.05, + 0.1, + 0.15000000000000002, + 0.2, + 0.25, + 0.30000000000000004, + 0.35000000000000003, + 0.4, + 0.45, + 0.5, + 0.55, + 0.6000000000000001, + 0.65, + 0.7000000000000001, + 0.75, + 0.8, + 0.8500000000000001, + 0.9, + 0.9500000000000001, + 1.0 + ], + "optimised_mse": [ + 0.02964925952255726, + 0.030000220984220505, + 0.03169703856110573, + 0.03395833075046539, + 0.03634364530444145, + 0.038573313504457474, + 0.04045669734477997, + 0.04184194654226303, + 0.04251442849636078, + 0.042437970638275146, + 0.04160228371620178, + 0.04010839760303497, + 0.03799746185541153, + 0.03541470691561699, + 0.032559722661972046, + 0.029640253633260727, + 0.026834338903427124, + 0.024286681786179543, + 0.02206854522228241, + 0.020447561517357826, + 0.020104100927710533 + ] +} \ No newline at end of file diff --git a/topk_k45_s1/runner_config.json b/topk_k45_s1/runner_config.json new file mode 100644 index 0000000000000000000000000000000000000000..adb72d14e7822bc83104b25e7d166bd5788f445d --- /dev/null +++ b/topk_k45_s1/runner_config.json @@ -0,0 +1,59 @@ +{ + "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1", + "sae": { + "d_in": 768, + "d_sae": 16384, + "dtype": "float32", + "device": "cuda", + "apply_b_dec_to_input": true, + "normalize_activations": "none", + "reshape_activations": "none", + "metadata": { + "sae_lens_version": "6.43.0", + "sae_lens_training_version": "6.43.0" + }, + "decoder_init_norm": 0.1, + "k": 45, + "use_sparse_activations": false, + "aux_loss_coefficient": 1.0, + "rescale_acts_by_decoder_norm": true, + "architecture": "topk" + }, + "training_samples": 50000000, + "batch_size": 4096, + "lr": 0.0003, + "lr_warm_up_steps": 1000, + "lr_decay_steps": 0, + "lr_scheduler_name": "constant", + "lr_end": 2.9999999999999997e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "n_restart_cycles": 1, + "device": "cuda", + "autocast_sae": false, + "autocast_data": false, + "n_checkpoints": 0, + "checkpoint_path": "runs/topk_k45_s1/ckpts", + "save_final_checkpoint": true, + "output_path": "runs/topk_k45_s1", + "save_synthetic_model": false, + "eval_frequency": 200000, + "eval_samples": 200000, + "run_final_eval": true, + "dead_feature_window": 1000, + "feature_sampling_window": 2000, + "n_batches_for_norm_estimate": 1000, + "sae_lens_version": "6.43.0", + "logger": { + "log_to_wandb": true, + "log_activations_store_to_wandb": false, + "log_optimizer_state_to_wandb": false, + "log_weights_to_wandb": true, + "wandb_project": "sae_lens_training", + "wandb_id": null, + "run_name": "synthetic-topk-16384-LR-0.0003", + "wandb_entity": null, + "wandb_log_frequency": 10, + "eval_every_n_wandb_logs": 100 + } +} \ No newline at end of file diff --git a/topk_k50_s1/barrier.json b/topk_k50_s1/barrier.json new file mode 100644 index 0000000000000000000000000000000000000000..4a4cfdb51a786a237a76fedc3f094bdf60293da4 --- /dev/null +++ b/topk_k50_s1/barrier.json @@ -0,0 +1,52 @@ +{ + "run": "runs/topk_k50_s1", + "k": 50, + "n_samples": 5000, + "aligned_cos_mean": 0.7114261984825134, + "alphas": [ + 0.0, + 0.05, + 0.1, + 0.15000000000000002, + 0.2, + 0.25, + 0.30000000000000004, + 0.35000000000000003, + 0.4, + 0.45, + 0.5, + 0.55, + 0.6000000000000001, + 0.65, + 0.7000000000000001, + 0.75, + 0.8, + 0.8500000000000001, + 0.9, + 0.9500000000000001, + 1.0 + ], + "mse": [ + 0.024120451882481575, + 0.02464943937957287, + 0.026377253234386444, + 0.02916892059147358, + 0.03298783302307129, + 0.03776884078979492, + 0.043275270611047745, + 0.04894585162401199, + 0.05390593409538269, + 0.057090096175670624, + 0.05774243548512459, + 0.055856186896562576, + 0.05165950953960419, + 0.045685287564992905, + 0.038359373807907104, + 0.030681241303682327, + 0.02351899817585945, + 0.017454368993639946, + 0.012889274396002293, + 0.010125655680894852, + 0.009323351085186005 + ] +} \ No newline at end of file diff --git a/topk_k50_s1/cfg.json b/topk_k50_s1/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..efae72ac7b4c98878699204dbe82c1b8188ad953 --- /dev/null +++ b/topk_k50_s1/cfg.json @@ -0,0 +1 @@ +{"device": "cuda", "reshape_activations": "none", "apply_b_dec_to_input": true, "dtype": "float32", "d_in": 768, "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "k": 50, "rescale_acts_by_decoder_norm": true, "normalize_activations": "none", "d_sae": 16384, "architecture": "topk"} \ No newline at end of file diff --git a/topk_k50_s1/ckpts/final_50003968/activation_scaler.json b/topk_k50_s1/ckpts/final_50003968/activation_scaler.json new file mode 100644 index 0000000000000000000000000000000000000000..5df4fdd0c0c44542391c33143ec07c0b30ec164d --- /dev/null +++ b/topk_k50_s1/ckpts/final_50003968/activation_scaler.json @@ -0,0 +1 @@ +{"scaling_factor": null} \ No newline at end of file diff --git a/topk_k50_s1/ckpts/final_50003968/cfg.json b/topk_k50_s1/ckpts/final_50003968/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..778e3c93c602e3ebb2509aad16cbeaf54dfd9b7f --- /dev/null +++ b/topk_k50_s1/ckpts/final_50003968/cfg.json @@ -0,0 +1 @@ +{"d_in": 768, "d_sae": 16384, "dtype": "float32", "device": "cuda", "apply_b_dec_to_input": true, "normalize_activations": "none", "reshape_activations": "none", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "decoder_init_norm": 0.1, "k": 50, "use_sparse_activations": false, "aux_loss_coefficient": 1.0, "rescale_acts_by_decoder_norm": true, "architecture": "topk"} \ No newline at end of file diff --git a/topk_k50_s1/ckpts/final_50003968/runner_config.json b/topk_k50_s1/ckpts/final_50003968/runner_config.json new file mode 100644 index 0000000000000000000000000000000000000000..4c2ad238ad978c6c4d1fb2f463ce8ec23fc9947f --- /dev/null +++ b/topk_k50_s1/ckpts/final_50003968/runner_config.json @@ -0,0 +1,59 @@ +{ + "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1", + "sae": { + "d_in": 768, + "d_sae": 16384, + "dtype": "float32", + "device": "cuda", + "apply_b_dec_to_input": true, + "normalize_activations": "none", + "reshape_activations": "none", + "metadata": { + "sae_lens_version": "6.43.0", + "sae_lens_training_version": "6.43.0" + }, + "decoder_init_norm": 0.1, + "k": 50, + "use_sparse_activations": false, + "aux_loss_coefficient": 1.0, + "rescale_acts_by_decoder_norm": true, + "architecture": "topk" + }, + "training_samples": 50000000, + "batch_size": 4096, + "lr": 0.0003, + "lr_warm_up_steps": 1000, + "lr_decay_steps": 0, + "lr_scheduler_name": "constant", + "lr_end": 2.9999999999999997e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "n_restart_cycles": 1, + "device": "cuda", + "autocast_sae": false, + "autocast_data": false, + "n_checkpoints": 0, + "checkpoint_path": "runs/topk_k50_s1/ckpts", + "save_final_checkpoint": true, + "output_path": "runs/topk_k50_s1", + "save_synthetic_model": false, + "eval_frequency": 200000, + "eval_samples": 200000, + "run_final_eval": true, + "dead_feature_window": 1000, + "feature_sampling_window": 2000, + "n_batches_for_norm_estimate": 1000, + "sae_lens_version": "6.43.0", + "logger": { + "log_to_wandb": true, + "log_activations_store_to_wandb": false, + "log_optimizer_state_to_wandb": false, + "log_weights_to_wandb": true, + "wandb_project": "sae_lens_training", + "wandb_id": null, + "run_name": "synthetic-topk-16384-LR-0.0003", + "wandb_entity": null, + "wandb_log_frequency": 10, + "eval_every_n_wandb_logs": 100 + } +} \ No newline at end of file diff --git a/topk_k50_s1/eval_stats.json b/topk_k50_s1/eval_stats.json new file mode 100644 index 0000000000000000000000000000000000000000..a8acf6098f14734dac677543facd8132ea7258d0 --- /dev/null +++ b/topk_k50_s1/eval_stats.json @@ -0,0 +1,15 @@ +{ + "true_l0": 34.512515, + "sae_l0": 50.0, + "dead_latents": 2208, + "shrinkage": 0.9621369616699219, + "explained_variance": 0.9290949695433401, + "mcc": 0.7136319875717163, + "uniqueness": 0.8148193359375, + "classification": { + "precision": 0.49263572692871094, + "recall": 0.7630173563957214, + "f1_score": 0.5833818912506104, + "accuracy": 0.9966489672660828 + } +} \ No newline at end of file diff --git a/topk_k50_s1/meta.json b/topk_k50_s1/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..d75276f5047cdb02a26c08be182e31f8c0b9523b --- /dev/null +++ b/topk_k50_s1/meta.json @@ -0,0 +1,9 @@ +{ + "variant": "topk", + "k": 50, + "l1": 5.0, + "seed": 1, + "n_samples": 50000000, + "seconds": 1494.6656107902527, + "out": "runs/topk_k50_s1" +} \ No newline at end of file diff --git a/topk_k50_s1/mode_connectivity.json b/topk_k50_s1/mode_connectivity.json new file mode 100644 index 0000000000000000000000000000000000000000..8ec668024a754ccc27509002fc8489b7265ddcd5 --- /dev/null +++ b/topk_k50_s1/mode_connectivity.json @@ -0,0 +1,57 @@ +{ + "run": "runs/topk_k50_s1", + "k": 50, + "train_alphas": [ + 0.2, + 0.4, + 0.5, + 0.6, + 0.8 + ], + "alphas": [ + 0.0, + 0.05, + 0.1, + 0.15000000000000002, + 0.2, + 0.25, + 0.30000000000000004, + 0.35000000000000003, + 0.4, + 0.45, + 0.5, + 0.55, + 0.6000000000000001, + 0.65, + 0.7000000000000001, + 0.75, + 0.8, + 0.8500000000000001, + 0.9, + 0.9500000000000001, + 1.0 + ], + "optimised_mse": [ + 0.024132613092660904, + 0.024458779022097588, + 0.026172030717134476, + 0.028542334213852882, + 0.0311331357806921, + 0.033607497811317444, + 0.03571644052863121, + 0.037315525114536285, + 0.0382796935737133, + 0.0384591706097126, + 0.03782765567302704, + 0.036362048238515854, + 0.03417735919356346, + 0.031407520174980164, + 0.028178105130791664, + 0.024620862677693367, + 0.020939022302627563, + 0.017227405682206154, + 0.013854061253368855, + 0.011224665679037571, + 0.010361848399043083 + ] +} \ No newline at end of file diff --git a/topk_k50_s1/runner_config.json b/topk_k50_s1/runner_config.json new file mode 100644 index 0000000000000000000000000000000000000000..4c2ad238ad978c6c4d1fb2f463ce8ec23fc9947f --- /dev/null +++ b/topk_k50_s1/runner_config.json @@ -0,0 +1,59 @@ +{ + "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1", + "sae": { + "d_in": 768, + "d_sae": 16384, + "dtype": "float32", + "device": "cuda", + "apply_b_dec_to_input": true, + "normalize_activations": "none", + "reshape_activations": "none", + "metadata": { + "sae_lens_version": "6.43.0", + "sae_lens_training_version": "6.43.0" + }, + "decoder_init_norm": 0.1, + "k": 50, + "use_sparse_activations": false, + "aux_loss_coefficient": 1.0, + "rescale_acts_by_decoder_norm": true, + "architecture": "topk" + }, + "training_samples": 50000000, + "batch_size": 4096, + "lr": 0.0003, + "lr_warm_up_steps": 1000, + "lr_decay_steps": 0, + "lr_scheduler_name": "constant", + "lr_end": 2.9999999999999997e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "n_restart_cycles": 1, + "device": "cuda", + "autocast_sae": false, + "autocast_data": false, + "n_checkpoints": 0, + "checkpoint_path": "runs/topk_k50_s1/ckpts", + "save_final_checkpoint": true, + "output_path": "runs/topk_k50_s1", + "save_synthetic_model": false, + "eval_frequency": 200000, + "eval_samples": 200000, + "run_final_eval": true, + "dead_feature_window": 1000, + "feature_sampling_window": 2000, + "n_batches_for_norm_estimate": 1000, + "sae_lens_version": "6.43.0", + "logger": { + "log_to_wandb": true, + "log_activations_store_to_wandb": false, + "log_optimizer_state_to_wandb": false, + "log_weights_to_wandb": true, + "wandb_project": "sae_lens_training", + "wandb_id": null, + "run_name": "synthetic-topk-16384-LR-0.0003", + "wandb_entity": null, + "wandb_log_frequency": 10, + "eval_every_n_wandb_logs": 100 + } +} \ No newline at end of file diff --git a/topk_k50_s2/barrier.json b/topk_k50_s2/barrier.json new file mode 100644 index 0000000000000000000000000000000000000000..1c6391249e8654606c12e2278b3e7ce159de39da --- /dev/null +++ b/topk_k50_s2/barrier.json @@ -0,0 +1,52 @@ +{ + "run": "runs/topk_k50_s2", + "k": 50, + "n_samples": 5000, + "aligned_cos_mean": 0.7091041803359985, + "alphas": [ + 0.0, + 0.05, + 0.1, + 0.15000000000000002, + 0.2, + 0.25, + 0.30000000000000004, + 0.35000000000000003, + 0.4, + 0.45, + 0.5, + 0.55, + 0.6000000000000001, + 0.65, + 0.7000000000000001, + 0.75, + 0.8, + 0.8500000000000001, + 0.9, + 0.9500000000000001, + 1.0 + ], + "mse": [ + 0.024920456111431122, + 0.025521647185087204, + 0.027605442330241203, + 0.031072892248630524, + 0.03572621941566467, + 0.041313085705041885, + 0.04735126346349716, + 0.05317096412181854, + 0.05782647430896759, + 0.06035330519080162, + 0.06024998426437378, + 0.05780115723609924, + 0.05330759659409523, + 0.04724709689617157, + 0.03983664512634277, + 0.03204873576760292, + 0.02483270689845085, + 0.01874225214123726, + 0.014218754135072231, + 0.0115033695474267, + 0.01074904389679432 + ] +} \ No newline at end of file diff --git a/topk_k50_s2/cfg.json b/topk_k50_s2/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..b0ea80f8699ae283f3bfdd034c4b0d1eed67cb06 --- /dev/null +++ b/topk_k50_s2/cfg.json @@ -0,0 +1 @@ +{"device": "cuda", "d_in": 768, "d_sae": 16384, "k": 50, "dtype": "float32", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "normalize_activations": "none", "reshape_activations": "none", "apply_b_dec_to_input": true, "rescale_acts_by_decoder_norm": true, "architecture": "topk"} \ No newline at end of file diff --git a/topk_k50_s2/ckpts/final_50003968/activation_scaler.json b/topk_k50_s2/ckpts/final_50003968/activation_scaler.json new file mode 100644 index 0000000000000000000000000000000000000000..5df4fdd0c0c44542391c33143ec07c0b30ec164d --- /dev/null +++ b/topk_k50_s2/ckpts/final_50003968/activation_scaler.json @@ -0,0 +1 @@ +{"scaling_factor": null} \ No newline at end of file diff --git a/topk_k50_s2/ckpts/final_50003968/cfg.json b/topk_k50_s2/ckpts/final_50003968/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..778e3c93c602e3ebb2509aad16cbeaf54dfd9b7f --- /dev/null +++ b/topk_k50_s2/ckpts/final_50003968/cfg.json @@ -0,0 +1 @@ +{"d_in": 768, "d_sae": 16384, "dtype": "float32", "device": "cuda", "apply_b_dec_to_input": true, "normalize_activations": "none", "reshape_activations": "none", "metadata": {"sae_lens_version": "6.43.0", "sae_lens_training_version": "6.43.0"}, "decoder_init_norm": 0.1, "k": 50, "use_sparse_activations": false, "aux_loss_coefficient": 1.0, "rescale_acts_by_decoder_norm": true, "architecture": "topk"} \ No newline at end of file diff --git a/topk_k50_s2/ckpts/final_50003968/runner_config.json b/topk_k50_s2/ckpts/final_50003968/runner_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f19c7ab4491d6251e1e14d7314ff0a142732c82a --- /dev/null +++ b/topk_k50_s2/ckpts/final_50003968/runner_config.json @@ -0,0 +1,59 @@ +{ + "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1", + "sae": { + "d_in": 768, + "d_sae": 16384, + "dtype": "float32", + "device": "cuda", + "apply_b_dec_to_input": true, + "normalize_activations": "none", + "reshape_activations": "none", + "metadata": { + "sae_lens_version": "6.43.0", + "sae_lens_training_version": "6.43.0" + }, + "decoder_init_norm": 0.1, + "k": 50, + "use_sparse_activations": false, + "aux_loss_coefficient": 1.0, + "rescale_acts_by_decoder_norm": true, + "architecture": "topk" + }, + "training_samples": 50000000, + "batch_size": 4096, + "lr": 0.0003, + "lr_warm_up_steps": 1000, + "lr_decay_steps": 0, + "lr_scheduler_name": "constant", + "lr_end": 2.9999999999999997e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "n_restart_cycles": 1, + "device": "cuda", + "autocast_sae": false, + "autocast_data": false, + "n_checkpoints": 0, + "checkpoint_path": "runs/topk_k50_s2/ckpts", + "save_final_checkpoint": true, + "output_path": "runs/topk_k50_s2", + "save_synthetic_model": false, + "eval_frequency": 200000, + "eval_samples": 200000, + "run_final_eval": true, + "dead_feature_window": 1000, + "feature_sampling_window": 2000, + "n_batches_for_norm_estimate": 1000, + "sae_lens_version": "6.43.0", + "logger": { + "log_to_wandb": true, + "log_activations_store_to_wandb": false, + "log_optimizer_state_to_wandb": false, + "log_weights_to_wandb": true, + "wandb_project": "sae_lens_training", + "wandb_id": null, + "run_name": "synthetic-topk-16384-LR-0.0003", + "wandb_entity": null, + "wandb_log_frequency": 10, + "eval_every_n_wandb_logs": 100 + } +} \ No newline at end of file diff --git a/topk_k50_s2/eval_stats.json b/topk_k50_s2/eval_stats.json new file mode 100644 index 0000000000000000000000000000000000000000..b9ca37cfa25bd755ced986c512c5e67ffa97145b --- /dev/null +++ b/topk_k50_s2/eval_stats.json @@ -0,0 +1,15 @@ +{ + "true_l0": 34.46119, + "sae_l0": 50.0, + "dead_latents": 2214, + "shrinkage": 0.9614712463378906, + "explained_variance": 0.9289294428204737, + "mcc": 0.7114471793174744, + "uniqueness": 0.81304931640625, + "classification": { + "precision": 0.4916253983974457, + "recall": 0.7625620365142822, + "f1_score": 0.5833791494369507, + "accuracy": 0.9965409636497498 + } +} \ No newline at end of file diff --git a/topk_k50_s2/meta.json b/topk_k50_s2/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..fecc109069e285110a9d4fe9bd4e9583d78d4d67 --- /dev/null +++ b/topk_k50_s2/meta.json @@ -0,0 +1,9 @@ +{ + "variant": "topk", + "k": 50, + "l1": 5.0, + "seed": 2, + "n_samples": 50000000, + "seconds": 1465.1242759227753, + "out": "runs/topk_k50_s2" +} \ No newline at end of file diff --git a/topk_k50_s2/mode_connectivity.json b/topk_k50_s2/mode_connectivity.json new file mode 100644 index 0000000000000000000000000000000000000000..3b4e68aa934e56df8c6d5d757697bd7843d74d7a --- /dev/null +++ b/topk_k50_s2/mode_connectivity.json @@ -0,0 +1,57 @@ +{ + "run": "runs/topk_k50_s2", + "k": 50, + "train_alphas": [ + 0.2, + 0.4, + 0.5, + 0.6, + 0.8 + ], + "alphas": [ + 0.0, + 0.05, + 0.1, + 0.15000000000000002, + 0.2, + 0.25, + 0.30000000000000004, + 0.35000000000000003, + 0.4, + 0.45, + 0.5, + 0.55, + 0.6000000000000001, + 0.65, + 0.7000000000000001, + 0.75, + 0.8, + 0.8500000000000001, + 0.9, + 0.9500000000000001, + 1.0 + ], + "optimised_mse": [ + 0.024101562798023224, + 0.024434832856059074, + 0.026152193546295166, + 0.02854280360043049, + 0.03113171085715294, + 0.03384040668606758, + 0.03786342963576317, + 0.03822896257042885, + 0.03821663558483124, + 0.038056112825870514, + 0.03739849478006363, + 0.036038972437381744, + 0.034020427614450455, + 0.03133762255311012, + 0.02818775735795498, + 0.024673402309417725, + 0.020953264087438583, + 0.017172399908304214, + 0.013675227761268616, + 0.01089582871645689, + 0.009933076798915863 + ] +} \ No newline at end of file diff --git a/topk_k50_s2/runner_config.json b/topk_k50_s2/runner_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f19c7ab4491d6251e1e14d7314ff0a142732c82a --- /dev/null +++ b/topk_k50_s2/runner_config.json @@ -0,0 +1,59 @@ +{ + "synthetic_model": "decoderesearch/synth-sae-bench-16k-v1", + "sae": { + "d_in": 768, + "d_sae": 16384, + "dtype": "float32", + "device": "cuda", + "apply_b_dec_to_input": true, + "normalize_activations": "none", + "reshape_activations": "none", + "metadata": { + "sae_lens_version": "6.43.0", + "sae_lens_training_version": "6.43.0" + }, + "decoder_init_norm": 0.1, + "k": 50, + "use_sparse_activations": false, + "aux_loss_coefficient": 1.0, + "rescale_acts_by_decoder_norm": true, + "architecture": "topk" + }, + "training_samples": 50000000, + "batch_size": 4096, + "lr": 0.0003, + "lr_warm_up_steps": 1000, + "lr_decay_steps": 0, + "lr_scheduler_name": "constant", + "lr_end": 2.9999999999999997e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "n_restart_cycles": 1, + "device": "cuda", + "autocast_sae": false, + "autocast_data": false, + "n_checkpoints": 0, + "checkpoint_path": "runs/topk_k50_s2/ckpts", + "save_final_checkpoint": true, + "output_path": "runs/topk_k50_s2", + "save_synthetic_model": false, + "eval_frequency": 200000, + "eval_samples": 200000, + "run_final_eval": true, + "dead_feature_window": 1000, + "feature_sampling_window": 2000, + "n_batches_for_norm_estimate": 1000, + "sae_lens_version": "6.43.0", + "logger": { + "log_to_wandb": true, + "log_activations_store_to_wandb": false, + "log_optimizer_state_to_wandb": false, + "log_weights_to_wandb": true, + "wandb_project": "sae_lens_training", + "wandb_id": null, + "run_name": "synthetic-topk-16384-LR-0.0003", + "wandb_entity": null, + "wandb_log_frequency": 10, + "eval_every_n_wandb_logs": 100 + } +} \ No newline at end of file